feat(ops): show worker and queue readiness
This commit is contained in:
@@ -7,6 +7,9 @@ from pathlib import Path
|
||||
from govoplan_core.core.operations import (
|
||||
OperationalCheck,
|
||||
OperationalCheckProviderRegistration,
|
||||
RuntimeWorkStatus,
|
||||
RuntimeWorkStatusContext,
|
||||
RuntimeWorkStatusProviderRegistration,
|
||||
)
|
||||
from govoplan_ops.backend.api.v1 import routes
|
||||
|
||||
@@ -14,6 +17,7 @@ from govoplan_ops.backend.api.v1 import routes
|
||||
@dataclass
|
||||
class _Manifest:
|
||||
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
|
||||
runtime_work_status_providers: tuple[RuntimeWorkStatusProviderRegistration, ...] = ()
|
||||
|
||||
|
||||
class _Registry:
|
||||
@@ -68,6 +72,98 @@ def test_module_operational_check_failure_is_isolated() -> None:
|
||||
assert "secret detail" not in result["detail"]
|
||||
|
||||
|
||||
def test_runtime_work_provider_cache_force_and_unknown_metrics() -> None:
|
||||
routes._runtime_work_cache.clear()
|
||||
calls = 0
|
||||
|
||||
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return RuntimeWorkStatus(
|
||||
provider_id="example.queue",
|
||||
label="Example queue",
|
||||
backend="Example",
|
||||
enabled=True,
|
||||
configured=True,
|
||||
state="healthy",
|
||||
detail="Workers answered; queue depth unsupported.",
|
||||
observed_at=context.observed_at,
|
||||
active_workers=1,
|
||||
queue_depths={"example": None},
|
||||
)
|
||||
|
||||
registry = _Registry()
|
||||
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
||||
RuntimeWorkStatusProviderRegistration(
|
||||
module_id="example",
|
||||
provider_id="example.queue",
|
||||
provider=provider,
|
||||
),
|
||||
)
|
||||
context = RuntimeWorkStatusContext(
|
||||
profile="split-worker",
|
||||
observed_at=datetime.now(UTC),
|
||||
stale_after_seconds=60,
|
||||
)
|
||||
|
||||
first = routes._runtime_work_statuses(registry, context)
|
||||
second = routes._runtime_work_statuses(registry, context)
|
||||
forced = routes._runtime_work_statuses(registry, context, force=True)
|
||||
|
||||
assert first[0]["queue_depths"] == {"example": None}
|
||||
assert second == first
|
||||
assert forced[0]["state"] == "healthy"
|
||||
assert calls == 2
|
||||
|
||||
|
||||
def test_runtime_work_provider_failure_is_sanitized() -> None:
|
||||
routes._runtime_work_cache.clear()
|
||||
|
||||
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
||||
del context
|
||||
raise RuntimeError("redis://user:secret@example.test")
|
||||
|
||||
registry = _Registry()
|
||||
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
||||
RuntimeWorkStatusProviderRegistration(
|
||||
module_id="example",
|
||||
provider_id="example.failed",
|
||||
provider=provider,
|
||||
),
|
||||
)
|
||||
result = routes._runtime_work_statuses(
|
||||
registry,
|
||||
RuntimeWorkStatusContext(
|
||||
profile="split-worker",
|
||||
observed_at=datetime.now(UTC),
|
||||
stale_after_seconds=60,
|
||||
),
|
||||
force=True,
|
||||
)[0]
|
||||
|
||||
assert result["state"] == "unreachable"
|
||||
assert "secret" not in result["detail"]
|
||||
|
||||
|
||||
def test_disabled_workers_are_expected_only_in_development() -> None:
|
||||
disabled = {
|
||||
"provider_id": "example.queue",
|
||||
"state": "disabled",
|
||||
"detail": "Intentionally disabled.",
|
||||
"enabled": False,
|
||||
"configured": True,
|
||||
"queue_depths": {},
|
||||
}
|
||||
|
||||
development = routes._runtime_work_check([disabled], "local-dev")
|
||||
production = routes._runtime_work_check([disabled], "single-process")
|
||||
|
||||
assert development["state"] == "inactive"
|
||||
assert development["readiness_critical"] is False
|
||||
assert production["state"] == "warning"
|
||||
assert production["readiness_critical"] is True
|
||||
|
||||
|
||||
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
|
||||
check = routes._runtime_cluster_check(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user