feat(ops): show worker and queue readiness

This commit is contained in:
2026-08-19 20:52:44 +02:00
parent f84097224b
commit 6ecb94c99e
12 changed files with 559 additions and 161 deletions
+96
View File
@@ -7,6 +7,9 @@ from pathlib import Path
from govoplan_core.core.operations import (
OperationalCheck,
OperationalCheckProviderRegistration,
RuntimeWorkStatus,
RuntimeWorkStatusContext,
RuntimeWorkStatusProviderRegistration,
)
from govoplan_ops.backend.api.v1 import routes
@@ -14,6 +17,7 @@ from govoplan_ops.backend.api.v1 import routes
@dataclass
class _Manifest:
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
runtime_work_status_providers: tuple[RuntimeWorkStatusProviderRegistration, ...] = ()
class _Registry:
@@ -68,6 +72,98 @@ def test_module_operational_check_failure_is_isolated() -> None:
assert "secret detail" not in result["detail"]
def test_runtime_work_provider_cache_force_and_unknown_metrics() -> None:
routes._runtime_work_cache.clear()
calls = 0
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
nonlocal calls
calls += 1
return RuntimeWorkStatus(
provider_id="example.queue",
label="Example queue",
backend="Example",
enabled=True,
configured=True,
state="healthy",
detail="Workers answered; queue depth unsupported.",
observed_at=context.observed_at,
active_workers=1,
queue_depths={"example": None},
)
registry = _Registry()
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
RuntimeWorkStatusProviderRegistration(
module_id="example",
provider_id="example.queue",
provider=provider,
),
)
context = RuntimeWorkStatusContext(
profile="split-worker",
observed_at=datetime.now(UTC),
stale_after_seconds=60,
)
first = routes._runtime_work_statuses(registry, context)
second = routes._runtime_work_statuses(registry, context)
forced = routes._runtime_work_statuses(registry, context, force=True)
assert first[0]["queue_depths"] == {"example": None}
assert second == first
assert forced[0]["state"] == "healthy"
assert calls == 2
def test_runtime_work_provider_failure_is_sanitized() -> None:
routes._runtime_work_cache.clear()
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
del context
raise RuntimeError("redis://user:secret@example.test")
registry = _Registry()
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
RuntimeWorkStatusProviderRegistration(
module_id="example",
provider_id="example.failed",
provider=provider,
),
)
result = routes._runtime_work_statuses(
registry,
RuntimeWorkStatusContext(
profile="split-worker",
observed_at=datetime.now(UTC),
stale_after_seconds=60,
),
force=True,
)[0]
assert result["state"] == "unreachable"
assert "secret" not in result["detail"]
def test_disabled_workers_are_expected_only_in_development() -> None:
disabled = {
"provider_id": "example.queue",
"state": "disabled",
"detail": "Intentionally disabled.",
"enabled": False,
"configured": True,
"queue_depths": {},
}
development = routes._runtime_work_check([disabled], "local-dev")
production = routes._runtime_work_check([disabled], "single-process")
assert development["state"] == "inactive"
assert development["readiness_critical"] is False
assert production["state"] == "warning"
assert production["readiness_critical"] is True
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
check = routes._runtime_cluster_check(
{