127 lines
4.0 KiB
Python
127 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from govoplan_core.core.operations import (
|
|
OperationalCheck,
|
|
OperationalCheckProviderRegistration,
|
|
)
|
|
from govoplan_ops.backend.api.v1 import routes
|
|
|
|
|
|
@dataclass
|
|
class _Manifest:
|
|
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
|
|
|
|
|
|
class _Registry:
|
|
def __init__(self, *registrations: OperationalCheckProviderRegistration):
|
|
self._manifest = _Manifest(tuple(registrations))
|
|
|
|
def manifests(self):
|
|
return (self._manifest,)
|
|
|
|
|
|
def test_module_operational_checks_cache_and_force() -> None:
|
|
routes._module_check_cache.clear()
|
|
calls = 0
|
|
|
|
def provider() -> OperationalCheck:
|
|
nonlocal calls
|
|
calls += 1
|
|
return OperationalCheck("example.roundtrip", "Example", "ok", "Passed")
|
|
|
|
registration = OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.roundtrip",
|
|
provider=provider,
|
|
)
|
|
registry = _Registry(registration)
|
|
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert calls == 1
|
|
routes._module_operational_checks(registry, force=True)
|
|
assert calls == 2
|
|
|
|
|
|
def test_module_operational_check_failure_is_isolated() -> None:
|
|
routes._module_check_cache.clear()
|
|
|
|
def provider() -> OperationalCheck:
|
|
raise RuntimeError("secret detail")
|
|
|
|
result = routes._module_operational_checks(
|
|
_Registry(
|
|
OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.failed",
|
|
provider=provider,
|
|
)
|
|
),
|
|
force=True,
|
|
)[0]
|
|
|
|
assert result["state"] == "error"
|
|
assert "secret detail" not in result["detail"]
|
|
|
|
|
|
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [
|
|
{"role": "api", "state": "active", "stale": False},
|
|
{"role": "worker", "state": "active", "stale": True},
|
|
],
|
|
"expected": {"api": 2, "worker": 1},
|
|
"active": {"api": 1, "worker": 0},
|
|
"recovery": {"requires_attention": 1},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["recovery_required"] == 1
|
|
|
|
|
|
def test_shared_runtime_cluster_skew_or_unserved_queue_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [],
|
|
"expected": {"api": 2, "worker": 2},
|
|
"active": {"api": 2, "worker": 2},
|
|
"composition": {"skewed": True},
|
|
"software_versions": {"skewed": False},
|
|
"queues": {"missing": ["calendar"]},
|
|
"recovery": {"requires_attention": 0},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["composition_skewed"] is True
|
|
assert check["metrics"]["missing_queues"] == 1
|
|
|
|
|
|
def test_database_capacity_check_enforces_shared_rendered_budget(
|
|
monkeypatch,
|
|
) -> None:
|
|
monkeypatch.setattr(routes.core_settings, "state_profile", "shared")
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_limit", 100)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_reserve", 10)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_available", 90)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 70)
|
|
|
|
healthy = routes._database_capacity_check()
|
|
assert healthy["state"] == "ok"
|
|
assert healthy["metrics"]["peak"] == 70
|
|
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 91)
|
|
overrun = routes._database_capacity_check()
|
|
assert overrun["state"] == "error"
|
|
assert overrun["readiness_critical"] is True
|