175 lines
5.1 KiB
Python
175 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from govoplan_core.core.modules import ModuleManifest
|
|
from govoplan_core.core.operations import (
|
|
RuntimeWorkStatusContext,
|
|
RuntimeWorkStatusProviderRegistration,
|
|
)
|
|
from govoplan_core.core.registry import PlatformRegistry, RegistryError
|
|
from govoplan_core.core.runtime_work import collect_celery_runtime_work_status
|
|
|
|
|
|
class Inspector:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
replies: dict[str, object] | None = None,
|
|
queues: dict[str, list[dict[str, str]]] | None = None,
|
|
active: dict[str, list[object]] | None = None,
|
|
reserved: dict[str, list[object]] | None = None,
|
|
) -> None:
|
|
self._replies = replies or {}
|
|
self._queues = queues or {}
|
|
self._active = active or {}
|
|
self._reserved = reserved or {}
|
|
|
|
def ping(self):
|
|
return self._replies
|
|
|
|
def active_queues(self):
|
|
return self._queues
|
|
|
|
def active(self):
|
|
return self._active
|
|
|
|
def reserved(self):
|
|
return self._reserved
|
|
|
|
|
|
def _context(*, stale: bool = False, worker: bool = True) -> RuntimeWorkStatusContext:
|
|
now = datetime.now(UTC)
|
|
nodes = (
|
|
{
|
|
"role": "worker",
|
|
"stale": stale,
|
|
"last_heartbeat_at": (now - timedelta(seconds=12)).isoformat(),
|
|
},
|
|
) if worker else ()
|
|
return RuntimeWorkStatusContext(
|
|
profile="production",
|
|
observed_at=now,
|
|
stale_after_seconds=60,
|
|
runtime_nodes=nodes,
|
|
)
|
|
|
|
|
|
def test_unknown_queue_depth_is_healthy_not_inferred_idle() -> None:
|
|
status = collect_celery_runtime_work_status(
|
|
_context(),
|
|
inspector=Inspector(
|
|
replies={"worker-1": {"ok": "pong"}},
|
|
queues={"worker-1": [{"name": "mail"}]},
|
|
),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {},
|
|
)
|
|
|
|
assert status.state == "healthy"
|
|
assert status.queue_depths == {"mail": None}
|
|
assert status.failures is None
|
|
assert status.last_heartbeat_at is not None
|
|
|
|
|
|
def test_busy_and_idle_require_measured_evidence() -> None:
|
|
busy = collect_celery_runtime_work_status(
|
|
_context(),
|
|
inspector=Inspector(
|
|
replies={"worker-1": {"ok": "pong"}},
|
|
queues={"worker-1": [{"name": "mail"}]},
|
|
active={"worker-1": [{"id": "task-1"}]},
|
|
),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {"mail": 2},
|
|
)
|
|
idle = collect_celery_runtime_work_status(
|
|
_context(),
|
|
inspector=Inspector(
|
|
replies={"worker-1": {"ok": "pong"}},
|
|
queues={"worker-1": [{"name": "mail"}]},
|
|
),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {"mail": 0},
|
|
)
|
|
|
|
assert busy.state == "busy"
|
|
assert busy.active_work == 1
|
|
assert idle.state == "idle"
|
|
|
|
|
|
def test_partial_and_stale_worker_evidence_are_not_healthy() -> None:
|
|
degraded = collect_celery_runtime_work_status(
|
|
_context(),
|
|
inspector=Inspector(
|
|
replies={"worker-1": {"ok": "pong"}},
|
|
queues={"worker-1": [{"name": "mail"}]},
|
|
),
|
|
queues=["mail", "calendar"],
|
|
queue_depth_reader=lambda queues: {queue: 0 for queue in queues},
|
|
)
|
|
stale = collect_celery_runtime_work_status(
|
|
_context(stale=True),
|
|
inspector=Inspector(
|
|
replies={"worker-1": {"ok": "pong"}},
|
|
queues={"worker-1": [{"name": "mail"}]},
|
|
),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {"mail": 0},
|
|
)
|
|
|
|
assert degraded.state == "degraded"
|
|
assert stale.state == "stale"
|
|
|
|
|
|
def test_no_reply_distinguishes_starting_from_unreachable() -> None:
|
|
starting = collect_celery_runtime_work_status(
|
|
_context(),
|
|
inspector=Inspector(),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {},
|
|
)
|
|
unreachable = collect_celery_runtime_work_status(
|
|
_context(worker=False),
|
|
inspector=Inspector(),
|
|
queues=["mail"],
|
|
queue_depth_reader=lambda queues: {},
|
|
)
|
|
|
|
assert starting.state == "starting"
|
|
assert unreachable.state == "unreachable"
|
|
|
|
|
|
def test_registry_rejects_duplicate_runtime_work_provider_ids() -> None:
|
|
def provider(context: RuntimeWorkStatusContext):
|
|
raise AssertionError(context)
|
|
|
|
def registration(module_id: str) -> RuntimeWorkStatusProviderRegistration:
|
|
return RuntimeWorkStatusProviderRegistration(
|
|
module_id=module_id,
|
|
provider_id="example.queue",
|
|
provider=provider,
|
|
)
|
|
registry = PlatformRegistry()
|
|
registry.register(
|
|
ModuleManifest(
|
|
id="alpha",
|
|
name="Alpha",
|
|
version="1",
|
|
runtime_work_status_providers=(registration("alpha"),),
|
|
)
|
|
)
|
|
registry.register(
|
|
ModuleManifest(
|
|
id="beta",
|
|
name="Beta",
|
|
version="1",
|
|
runtime_work_status_providers=(registration("beta"),),
|
|
)
|
|
)
|
|
|
|
with pytest.raises(RegistryError, match="Duplicate runtime-work provider"):
|
|
registry.validate()
|