289 lines
8.5 KiB
Python
289 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from app.config import settings
|
|
|
|
|
|
@dataclass
|
|
class WorkerHandle:
|
|
index: int
|
|
worker_id: str
|
|
pid: int | None
|
|
status: str
|
|
pid_file: Path
|
|
log_file: Path
|
|
started_by_server: bool = False
|
|
|
|
|
|
_handles: list[WorkerHandle] = []
|
|
|
|
|
|
def start_queue_workers(*, force: bool = False) -> list[WorkerHandle]:
|
|
if not force and not settings.queue_worker_autostart:
|
|
return []
|
|
worker_count = max(0, int(settings.queue_worker_count))
|
|
handles: list[WorkerHandle] = []
|
|
worker_dir = settings.data_dir / "workers"
|
|
worker_dir.mkdir(parents=True, exist_ok=True)
|
|
for index in range(worker_count):
|
|
worker_id = f"server-worker-{index + 1}"
|
|
pid_file = worker_dir / f"{worker_id}.pid"
|
|
log_file = worker_dir / f"{worker_id}.log"
|
|
existing_pid = _read_pid(pid_file)
|
|
if existing_pid is not None and _worker_pid_running(existing_pid, worker_id):
|
|
handles.append(
|
|
WorkerHandle(
|
|
index=index,
|
|
worker_id=worker_id,
|
|
pid=existing_pid,
|
|
status="already_running",
|
|
pid_file=pid_file,
|
|
log_file=log_file,
|
|
)
|
|
)
|
|
continue
|
|
pid_file.unlink(missing_ok=True)
|
|
process = _spawn_worker(worker_id, log_file)
|
|
pid_file.write_text(str(process.pid), encoding="utf-8")
|
|
handles.append(
|
|
WorkerHandle(
|
|
index=index,
|
|
worker_id=worker_id,
|
|
pid=process.pid,
|
|
status="started",
|
|
pid_file=pid_file,
|
|
log_file=log_file,
|
|
started_by_server=True,
|
|
)
|
|
)
|
|
_handles[:] = handles
|
|
return list(_handles)
|
|
|
|
|
|
def stop_queue_workers() -> None:
|
|
if not settings.queue_worker_stop_on_shutdown:
|
|
return
|
|
stop_configured_queue_workers()
|
|
|
|
|
|
def stop_configured_queue_workers() -> list[WorkerHandle]:
|
|
handles: list[WorkerHandle] = []
|
|
worker_dir = settings.data_dir / "workers"
|
|
worker_dir.mkdir(parents=True, exist_ok=True)
|
|
worker_count = max(0, int(settings.queue_worker_count))
|
|
for index in range(worker_count):
|
|
worker_id = f"server-worker-{index + 1}"
|
|
pid_file = worker_dir / f"{worker_id}.pid"
|
|
log_file = worker_dir / f"{worker_id}.log"
|
|
pid = _read_pid(pid_file)
|
|
if pid is None:
|
|
handles.append(
|
|
WorkerHandle(
|
|
index=index,
|
|
worker_id=worker_id,
|
|
pid=None,
|
|
status="already_stopped",
|
|
pid_file=pid_file,
|
|
log_file=log_file,
|
|
)
|
|
)
|
|
continue
|
|
if not _worker_pid_running(pid, worker_id):
|
|
pid_file.unlink(missing_ok=True)
|
|
handles.append(
|
|
WorkerHandle(
|
|
index=index,
|
|
worker_id=worker_id,
|
|
pid=pid,
|
|
status="stale_pid_removed",
|
|
pid_file=pid_file,
|
|
log_file=log_file,
|
|
)
|
|
)
|
|
continue
|
|
stopped = _terminate_pid(pid)
|
|
if stopped:
|
|
pid_file.unlink(missing_ok=True)
|
|
status = "stopped"
|
|
stored_pid = None
|
|
else:
|
|
status = "stop_requested"
|
|
stored_pid = pid
|
|
handles.append(
|
|
WorkerHandle(
|
|
index=index,
|
|
worker_id=worker_id,
|
|
pid=stored_pid,
|
|
status=status,
|
|
pid_file=pid_file,
|
|
log_file=log_file,
|
|
)
|
|
)
|
|
_handles[:] = [
|
|
handle
|
|
for handle in _handles
|
|
if handle.pid is not None and _worker_pid_running(handle.pid, handle.worker_id)
|
|
]
|
|
return handles
|
|
|
|
|
|
def restart_queue_workers() -> dict[str, list[WorkerHandle]]:
|
|
stopped = stop_configured_queue_workers()
|
|
started = start_queue_workers(force=True)
|
|
return {"stopped": stopped, "started": started}
|
|
|
|
|
|
def queue_worker_status() -> list[dict[str, object]]:
|
|
worker_dir = settings.data_dir / "workers"
|
|
statuses: list[dict[str, object]] = []
|
|
configured_count = max(0, int(settings.queue_worker_count))
|
|
for index in range(configured_count):
|
|
worker_id = f"server-worker-{index + 1}"
|
|
pid_file = worker_dir / f"{worker_id}.pid"
|
|
log_file = worker_dir / f"{worker_id}.log"
|
|
pid = _read_pid(pid_file)
|
|
pid_alive = pid is not None and _pid_running(pid)
|
|
running = pid is not None and _worker_pid_running(pid, worker_id)
|
|
stale = pid is not None and not running
|
|
status = "running" if running else "stale" if stale else "stopped"
|
|
if pid_alive and stale:
|
|
status = "stale_pid_reused"
|
|
statuses.append(
|
|
{
|
|
"index": index,
|
|
"worker_id": worker_id,
|
|
"pid": pid,
|
|
"running": running,
|
|
"stale": stale,
|
|
"status": status,
|
|
"pid_file": str(pid_file),
|
|
"log_file": str(log_file),
|
|
}
|
|
)
|
|
return statuses
|
|
|
|
|
|
def _spawn_worker(worker_id: str, log_file: Path) -> subprocess.Popen:
|
|
root = Path(__file__).resolve().parents[1]
|
|
command = [
|
|
sys.executable,
|
|
"-m",
|
|
"app.cli",
|
|
"worker",
|
|
"--worker-id",
|
|
worker_id,
|
|
"--poll-interval",
|
|
str(settings.queue_worker_poll_interval_seconds),
|
|
]
|
|
env = os.environ.copy()
|
|
env["MOBILITY_SUPERVISED_WORKER"] = "1"
|
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
log_handle = log_file.open("ab", buffering=0)
|
|
try:
|
|
return subprocess.Popen(
|
|
command,
|
|
cwd=str(root),
|
|
env=env,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=log_handle,
|
|
stderr=subprocess.STDOUT,
|
|
start_new_session=True,
|
|
)
|
|
finally:
|
|
log_handle.close()
|
|
|
|
|
|
def _read_pid(path: Path) -> int | None:
|
|
try:
|
|
return int(path.read_text(encoding="utf-8").strip())
|
|
except (FileNotFoundError, ValueError, OSError):
|
|
return None
|
|
|
|
|
|
def _pid_running(pid: int) -> bool:
|
|
try:
|
|
os.kill(pid, 0)
|
|
except ProcessLookupError:
|
|
return False
|
|
except PermissionError:
|
|
return True
|
|
return not _pid_is_zombie(pid)
|
|
|
|
|
|
def _pid_is_zombie(pid: int) -> bool:
|
|
try:
|
|
for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8", errors="replace").splitlines():
|
|
if line.startswith("State:"):
|
|
return "\tZ" in line or "zombie" in line.lower()
|
|
except OSError:
|
|
return False
|
|
return False
|
|
|
|
|
|
def _worker_pid_running(pid: int, worker_id: str) -> bool:
|
|
return _pid_running(pid) and _pid_matches_worker(pid, worker_id)
|
|
|
|
|
|
def _pid_matches_worker(pid: int, worker_id: str) -> bool:
|
|
try:
|
|
raw_parts = Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0")
|
|
except (FileNotFoundError, ProcessLookupError, PermissionError, OSError):
|
|
return False
|
|
parts = [part.decode("utf-8", errors="replace") for part in raw_parts if part]
|
|
if "app.cli" not in parts or "worker" not in parts:
|
|
return False
|
|
try:
|
|
worker_id_index = parts.index("--worker-id") + 1
|
|
except ValueError:
|
|
return False
|
|
return worker_id_index < len(parts) and parts[worker_id_index] == worker_id
|
|
|
|
|
|
def _terminate_pid(pid: int, *, timeout_seconds: float = 5.0) -> bool:
|
|
_signal_pid(pid, signal.SIGTERM)
|
|
if _wait_for_exit(pid, timeout_seconds):
|
|
return True
|
|
_signal_pid(pid, signal.SIGKILL)
|
|
return _wait_for_exit(pid, 1.0)
|
|
|
|
|
|
def _signal_pid(pid: int, sig: signal.Signals) -> None:
|
|
try:
|
|
os.killpg(pid, sig)
|
|
except ProcessLookupError:
|
|
return
|
|
except OSError:
|
|
try:
|
|
os.kill(pid, sig)
|
|
except ProcessLookupError:
|
|
return
|
|
|
|
|
|
def _wait_for_exit(pid: int, timeout_seconds: float) -> bool:
|
|
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
|
while time.monotonic() < deadline:
|
|
if not _pid_running(pid):
|
|
_reap_pid(pid)
|
|
return True
|
|
time.sleep(0.1)
|
|
if not _pid_running(pid):
|
|
_reap_pid(pid)
|
|
return True
|
|
return False
|
|
|
|
|
|
def _reap_pid(pid: int) -> None:
|
|
try:
|
|
os.waitpid(pid, os.WNOHANG)
|
|
except ChildProcessError:
|
|
return
|
|
except OSError:
|
|
return
|