feat(operations): add runtime work status contract
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest
|
||||
from govoplan_core.core.operations import (
|
||||
RuntimeWorkStatusContext,
|
||||
RuntimeWorkStatusProviderRegistration,
|
||||
)
|
||||
from govoplan_core.core.registry import PlatformRegistry, RegistryError
|
||||
from govoplan_core.core.runtime_work import collect_celery_runtime_work_status
|
||||
|
||||
|
||||
class Inspector:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
replies: dict[str, object] | None = None,
|
||||
queues: dict[str, list[dict[str, str]]] | None = None,
|
||||
active: dict[str, list[object]] | None = None,
|
||||
reserved: dict[str, list[object]] | None = None,
|
||||
) -> None:
|
||||
self._replies = replies or {}
|
||||
self._queues = queues or {}
|
||||
self._active = active or {}
|
||||
self._reserved = reserved or {}
|
||||
|
||||
def ping(self):
|
||||
return self._replies
|
||||
|
||||
def active_queues(self):
|
||||
return self._queues
|
||||
|
||||
def active(self):
|
||||
return self._active
|
||||
|
||||
def reserved(self):
|
||||
return self._reserved
|
||||
|
||||
|
||||
def _context(*, stale: bool = False, worker: bool = True) -> RuntimeWorkStatusContext:
|
||||
now = datetime.now(UTC)
|
||||
nodes = (
|
||||
{
|
||||
"role": "worker",
|
||||
"stale": stale,
|
||||
"last_heartbeat_at": (now - timedelta(seconds=12)).isoformat(),
|
||||
},
|
||||
) if worker else ()
|
||||
return RuntimeWorkStatusContext(
|
||||
profile="production",
|
||||
observed_at=now,
|
||||
stale_after_seconds=60,
|
||||
runtime_nodes=nodes,
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_queue_depth_is_healthy_not_inferred_idle() -> None:
|
||||
status = collect_celery_runtime_work_status(
|
||||
_context(),
|
||||
inspector=Inspector(
|
||||
replies={"worker-1": {"ok": "pong"}},
|
||||
queues={"worker-1": [{"name": "mail"}]},
|
||||
),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {},
|
||||
)
|
||||
|
||||
assert status.state == "healthy"
|
||||
assert status.queue_depths == {"mail": None}
|
||||
assert status.failures is None
|
||||
assert status.last_heartbeat_at is not None
|
||||
|
||||
|
||||
def test_busy_and_idle_require_measured_evidence() -> None:
|
||||
busy = collect_celery_runtime_work_status(
|
||||
_context(),
|
||||
inspector=Inspector(
|
||||
replies={"worker-1": {"ok": "pong"}},
|
||||
queues={"worker-1": [{"name": "mail"}]},
|
||||
active={"worker-1": [{"id": "task-1"}]},
|
||||
),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {"mail": 2},
|
||||
)
|
||||
idle = collect_celery_runtime_work_status(
|
||||
_context(),
|
||||
inspector=Inspector(
|
||||
replies={"worker-1": {"ok": "pong"}},
|
||||
queues={"worker-1": [{"name": "mail"}]},
|
||||
),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {"mail": 0},
|
||||
)
|
||||
|
||||
assert busy.state == "busy"
|
||||
assert busy.active_work == 1
|
||||
assert idle.state == "idle"
|
||||
|
||||
|
||||
def test_partial_and_stale_worker_evidence_are_not_healthy() -> None:
|
||||
degraded = collect_celery_runtime_work_status(
|
||||
_context(),
|
||||
inspector=Inspector(
|
||||
replies={"worker-1": {"ok": "pong"}},
|
||||
queues={"worker-1": [{"name": "mail"}]},
|
||||
),
|
||||
queues=["mail", "calendar"],
|
||||
queue_depth_reader=lambda queues: {queue: 0 for queue in queues},
|
||||
)
|
||||
stale = collect_celery_runtime_work_status(
|
||||
_context(stale=True),
|
||||
inspector=Inspector(
|
||||
replies={"worker-1": {"ok": "pong"}},
|
||||
queues={"worker-1": [{"name": "mail"}]},
|
||||
),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {"mail": 0},
|
||||
)
|
||||
|
||||
assert degraded.state == "degraded"
|
||||
assert stale.state == "stale"
|
||||
|
||||
|
||||
def test_no_reply_distinguishes_starting_from_unreachable() -> None:
|
||||
starting = collect_celery_runtime_work_status(
|
||||
_context(),
|
||||
inspector=Inspector(),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {},
|
||||
)
|
||||
unreachable = collect_celery_runtime_work_status(
|
||||
_context(worker=False),
|
||||
inspector=Inspector(),
|
||||
queues=["mail"],
|
||||
queue_depth_reader=lambda queues: {},
|
||||
)
|
||||
|
||||
assert starting.state == "starting"
|
||||
assert unreachable.state == "unreachable"
|
||||
|
||||
|
||||
def test_registry_rejects_duplicate_runtime_work_provider_ids() -> None:
|
||||
def provider(context: RuntimeWorkStatusContext):
|
||||
raise AssertionError(context)
|
||||
|
||||
def registration(module_id: str) -> RuntimeWorkStatusProviderRegistration:
|
||||
return RuntimeWorkStatusProviderRegistration(
|
||||
module_id=module_id,
|
||||
provider_id="example.queue",
|
||||
provider=provider,
|
||||
)
|
||||
registry = PlatformRegistry()
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
id="alpha",
|
||||
name="Alpha",
|
||||
version="1",
|
||||
runtime_work_status_providers=(registration("alpha"),),
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
id="beta",
|
||||
name="Beta",
|
||||
version="1",
|
||||
runtime_work_status_providers=(registration("beta"),),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(RegistryError, match="Duplicate runtime-work provider"):
|
||||
registry.validate()
|
||||
Reference in New Issue
Block a user