Next alpha stage commit

app/worker_supervisor.py

@@ -4,6 +4,7 @@ import os
 import signal
 import subprocess
 import sys
+import time
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -24,8 +25,8 @@ class WorkerHandle:
 _handles: list[WorkerHandle] = []
 
 
-def start_queue_workers() -> list[WorkerHandle]:
-    if not settings.queue_worker_autostart:
+def start_queue_workers(*, force: bool = False) -> list[WorkerHandle]:
+    if not force and not settings.queue_worker_autostart:
         return []
     worker_count = max(0, int(settings.queue_worker_count))
     handles: list[WorkerHandle] = []
@@ -36,7 +37,7 @@ def start_queue_workers() -> list[WorkerHandle]:
         pid_file = worker_dir / f"{worker_id}.pid"
         log_file = worker_dir / f"{worker_id}.log"
         existing_pid = _read_pid(pid_file)
-        if existing_pid is not None and _pid_running(existing_pid):
+        if existing_pid is not None and _worker_pid_running(existing_pid, worker_id):
             handles.append(
                 WorkerHandle(
                     index=index,
@@ -69,16 +70,77 @@ def start_queue_workers() -> list[WorkerHandle]:
 def stop_queue_workers() -> None:
     if not settings.queue_worker_stop_on_shutdown:
         return
-    for handle in list(_handles):
-        if not handle.started_by_server or handle.pid is None:
+    stop_configured_queue_workers()
+
+
+def stop_configured_queue_workers() -> list[WorkerHandle]:
+    handles: list[WorkerHandle] = []
+    worker_dir = settings.data_dir / "workers"
+    worker_dir.mkdir(parents=True, exist_ok=True)
+    worker_count = max(0, int(settings.queue_worker_count))
+    for index in range(worker_count):
+        worker_id = f"server-worker-{index + 1}"
+        pid_file = worker_dir / f"{worker_id}.pid"
+        log_file = worker_dir / f"{worker_id}.log"
+        pid = _read_pid(pid_file)
+        if pid is None:
+            handles.append(
+                WorkerHandle(
+                    index=index,
+                    worker_id=worker_id,
+                    pid=None,
+                    status="already_stopped",
+                    pid_file=pid_file,
+                    log_file=log_file,
+                )
+            )
             continue
-        _terminate_pid(handle.pid)
-        handle.pid_file.unlink(missing_ok=True)
+        if not _worker_pid_running(pid, worker_id):
+            pid_file.unlink(missing_ok=True)
+            handles.append(
+                WorkerHandle(
+                    index=index,
+                    worker_id=worker_id,
+                    pid=pid,
+                    status="stale_pid_removed",
+                    pid_file=pid_file,
+                    log_file=log_file,
+                )
+            )
+            continue
+        stopped = _terminate_pid(pid)
+        if stopped:
+            pid_file.unlink(missing_ok=True)
+            status = "stopped"
+            stored_pid = None
+        else:
+            status = "stop_requested"
+            stored_pid = pid
+        handles.append(
+            WorkerHandle(
+                index=index,
+                worker_id=worker_id,
+                pid=stored_pid,
+                status=status,
+                pid_file=pid_file,
+                log_file=log_file,
+            )
+        )
+    _handles[:] = [
+        handle
+        for handle in _handles
+        if handle.pid is not None and _worker_pid_running(handle.pid, handle.worker_id)
+    ]
+    return handles
+
+
+def restart_queue_workers() -> dict[str, list[WorkerHandle]]:
+    stopped = stop_configured_queue_workers()
+    started = start_queue_workers(force=True)
+    return {"stopped": stopped, "started": started}
 
 
 def queue_worker_status() -> list[dict[str, object]]:
-    if not settings.queue_worker_autostart:
-        return []
     worker_dir = settings.data_dir / "workers"
     statuses: list[dict[str, object]] = []
     configured_count = max(0, int(settings.queue_worker_count))
@@ -87,13 +149,20 @@ def queue_worker_status() -> list[dict[str, object]]:
         pid_file = worker_dir / f"{worker_id}.pid"
         log_file = worker_dir / f"{worker_id}.log"
         pid = _read_pid(pid_file)
-        running = pid is not None and _pid_running(pid)
+        pid_alive = pid is not None and _pid_running(pid)
+        running = pid is not None and _worker_pid_running(pid, worker_id)
+        stale = pid is not None and not running
+        status = "running" if running else "stale" if stale else "stopped"
+        if pid_alive and stale:
+            status = "stale_pid_reused"
         statuses.append(
             {
                 "index": index,
                 "worker_id": worker_id,
                 "pid": pid,
                 "running": running,
+                "stale": stale,
+                "status": status,
                 "pid_file": str(pid_file),
                 "log_file": str(log_file),
             }
@@ -145,11 +214,75 @@ def _pid_running(pid: int) -> bool:
         return False
     except PermissionError:
         return True
-    return True
+    return not _pid_is_zombie(pid)
 
 
-def _terminate_pid(pid: int) -> None:
+def _pid_is_zombie(pid: int) -> bool:
     try:
-        os.kill(pid, signal.SIGTERM)
+        for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8", errors="replace").splitlines():
+            if line.startswith("State:"):
+                return "\tZ" in line or "zombie" in line.lower()
+    except OSError:
+        return False
+    return False
+
+
+def _worker_pid_running(pid: int, worker_id: str) -> bool:
+    return _pid_running(pid) and _pid_matches_worker(pid, worker_id)
+
+
+def _pid_matches_worker(pid: int, worker_id: str) -> bool:
+    try:
+        raw_parts = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0")
+    except (FileNotFoundError, ProcessLookupError, PermissionError, OSError):
+        return False
+    parts = [part.decode("utf-8", errors="replace") for part in raw_parts if part]
+    if "app.cli" not in parts or "worker" not in parts:
+        return False
+    try:
+        worker_id_index = parts.index("--worker-id") + 1
+    except ValueError:
+        return False
+    return worker_id_index < len(parts) and parts[worker_id_index] == worker_id
+
+
+def _terminate_pid(pid: int, *, timeout_seconds: float = 5.0) -> bool:
+    _signal_pid(pid, signal.SIGTERM)
+    if _wait_for_exit(pid, timeout_seconds):
+        return True
+    _signal_pid(pid, signal.SIGKILL)
+    return _wait_for_exit(pid, 1.0)
+
+
+def _signal_pid(pid: int, sig: signal.Signals) -> None:
+    try:
+        os.killpg(pid, sig)
     except ProcessLookupError:
         return
+    except OSError:
+        try:
+            os.kill(pid, sig)
+        except ProcessLookupError:
+            return
+
+
+def _wait_for_exit(pid: int, timeout_seconds: float) -> bool:
+    deadline = time.monotonic() + max(0.0, timeout_seconds)
+    while time.monotonic() < deadline:
+        if not _pid_running(pid):
+            _reap_pid(pid)
+            return True
+        time.sleep(0.1)
+    if not _pid_running(pid):
+        _reap_pid(pid)
+        return True
+    return False
+
+
+def _reap_pid(pid: int) -> None:
+    try:
+        os.waitpid(pid, os.WNOHANG)
+    except ChildProcessError:
+        return
+    except OSError:
+        return