from __future__ import annotations import os import signal import subprocess import sys import time from dataclasses import dataclass from pathlib import Path from app.config import settings @dataclass class WorkerHandle: index: int worker_id: str pid: int | None status: str pid_file: Path log_file: Path started_by_server: bool = False _handles: list[WorkerHandle] = [] def start_queue_workers(*, force: bool = False) -> list[WorkerHandle]: if not force and not settings.queue_worker_autostart: return [] worker_count = max(0, int(settings.queue_worker_count)) handles: list[WorkerHandle] = [] worker_dir = settings.data_dir / "workers" worker_dir.mkdir(parents=True, exist_ok=True) for index in range(worker_count): worker_id = f"server-worker-{index + 1}" pid_file = worker_dir / f"{worker_id}.pid" log_file = worker_dir / f"{worker_id}.log" existing_pid = _read_pid(pid_file) if existing_pid is not None and _worker_pid_running(existing_pid, worker_id): handles.append( WorkerHandle( index=index, worker_id=worker_id, pid=existing_pid, status="already_running", pid_file=pid_file, log_file=log_file, ) ) continue pid_file.unlink(missing_ok=True) process = _spawn_worker(worker_id, log_file) pid_file.write_text(str(process.pid), encoding="utf-8") handles.append( WorkerHandle( index=index, worker_id=worker_id, pid=process.pid, status="started", pid_file=pid_file, log_file=log_file, started_by_server=True, ) ) _handles[:] = handles return list(_handles) def stop_queue_workers() -> None: if not settings.queue_worker_stop_on_shutdown: return stop_configured_queue_workers() def stop_configured_queue_workers() -> list[WorkerHandle]: handles: list[WorkerHandle] = [] worker_dir = settings.data_dir / "workers" worker_dir.mkdir(parents=True, exist_ok=True) worker_count = max(0, int(settings.queue_worker_count)) for index in range(worker_count): worker_id = f"server-worker-{index + 1}" pid_file = worker_dir / f"{worker_id}.pid" log_file = worker_dir / f"{worker_id}.log" pid = _read_pid(pid_file) if pid is None: handles.append( WorkerHandle( index=index, worker_id=worker_id, pid=None, status="already_stopped", pid_file=pid_file, log_file=log_file, ) ) continue if not _worker_pid_running(pid, worker_id): pid_file.unlink(missing_ok=True) handles.append( WorkerHandle( index=index, worker_id=worker_id, pid=pid, status="stale_pid_removed", pid_file=pid_file, log_file=log_file, ) ) continue stopped = _terminate_pid(pid) if stopped: pid_file.unlink(missing_ok=True) status = "stopped" stored_pid = None else: status = "stop_requested" stored_pid = pid handles.append( WorkerHandle( index=index, worker_id=worker_id, pid=stored_pid, status=status, pid_file=pid_file, log_file=log_file, ) ) _handles[:] = [ handle for handle in _handles if handle.pid is not None and _worker_pid_running(handle.pid, handle.worker_id) ] return handles def restart_queue_workers() -> dict[str, list[WorkerHandle]]: stopped = stop_configured_queue_workers() started = start_queue_workers(force=True) return {"stopped": stopped, "started": started} def queue_worker_status() -> list[dict[str, object]]: worker_dir = settings.data_dir / "workers" statuses: list[dict[str, object]] = [] configured_count = max(0, int(settings.queue_worker_count)) for index in range(configured_count): worker_id = f"server-worker-{index + 1}" pid_file = worker_dir / f"{worker_id}.pid" log_file = worker_dir / f"{worker_id}.log" pid = _read_pid(pid_file) pid_alive = pid is not None and _pid_running(pid) running = pid is not None and _worker_pid_running(pid, worker_id) stale = pid is not None and not running status = "running" if running else "stale" if stale else "stopped" if pid_alive and stale: status = "stale_pid_reused" statuses.append( { "index": index, "worker_id": worker_id, "pid": pid, "running": running, "stale": stale, "status": status, "pid_file": str(pid_file), "log_file": str(log_file), } ) return statuses def _spawn_worker(worker_id: str, log_file: Path) -> subprocess.Popen: root = Path(__file__).resolve().parents[1] command = [ sys.executable, "-m", "app.cli", "worker", "--worker-id", worker_id, "--poll-interval", str(settings.queue_worker_poll_interval_seconds), ] env = os.environ.copy() env["MOBILITY_SUPERVISED_WORKER"] = "1" log_file.parent.mkdir(parents=True, exist_ok=True) log_handle = log_file.open("ab", buffering=0) try: return subprocess.Popen( command, cwd=str(root), env=env, stdin=subprocess.DEVNULL, stdout=log_handle, stderr=subprocess.STDOUT, start_new_session=True, ) finally: log_handle.close() def _read_pid(path: Path) -> int | None: try: return int(path.read_text(encoding="utf-8").strip()) except (FileNotFoundError, ValueError, OSError): return None def _pid_running(pid: int) -> bool: try: os.kill(pid, 0) except ProcessLookupError: return False except PermissionError: return True return not _pid_is_zombie(pid) def _pid_is_zombie(pid: int) -> bool: try: for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8", errors="replace").splitlines(): if line.startswith("State:"): return "\tZ" in line or "zombie" in line.lower() except OSError: return False return False def _worker_pid_running(pid: int, worker_id: str) -> bool: return _pid_running(pid) and _pid_matches_worker(pid, worker_id) def _pid_matches_worker(pid: int, worker_id: str) -> bool: try: raw_parts = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0") except (FileNotFoundError, ProcessLookupError, PermissionError, OSError): return False parts = [part.decode("utf-8", errors="replace") for part in raw_parts if part] if "app.cli" not in parts or "worker" not in parts: return False try: worker_id_index = parts.index("--worker-id") + 1 except ValueError: return False return worker_id_index < len(parts) and parts[worker_id_index] == worker_id def _terminate_pid(pid: int, *, timeout_seconds: float = 5.0) -> bool: _signal_pid(pid, signal.SIGTERM) if _wait_for_exit(pid, timeout_seconds): return True _signal_pid(pid, signal.SIGKILL) return _wait_for_exit(pid, 1.0) def _signal_pid(pid: int, sig: signal.Signals) -> None: try: os.killpg(pid, sig) except ProcessLookupError: return except OSError: try: os.kill(pid, sig) except ProcessLookupError: return def _wait_for_exit(pid: int, timeout_seconds: float) -> bool: deadline = time.monotonic() + max(0.0, timeout_seconds) while time.monotonic() < deadline: if not _pid_running(pid): _reap_pid(pid) return True time.sleep(0.1) if not _pid_running(pid): _reap_pid(pid) return True return False def _reap_pid(pid: int) -> None: try: os.waitpid(pid, os.WNOHANG) except ChildProcessError: return except OSError: return