159 lines
4.9 KiB
Python
159 lines
4.9 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from govoplan_core.core.operations import (
|
|
OperationalCheck,
|
|
OperationalCheckProviderRegistration,
|
|
)
|
|
from govoplan_ops.backend.api.v1 import routes
|
|
|
|
|
|
@dataclass
|
|
class _Manifest:
|
|
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
|
|
|
|
|
|
class _Registry:
|
|
def __init__(self, *registrations: OperationalCheckProviderRegistration):
|
|
self._manifest = _Manifest(tuple(registrations))
|
|
|
|
def manifests(self):
|
|
return (self._manifest,)
|
|
|
|
|
|
def test_module_operational_checks_cache_and_force() -> None:
|
|
routes._module_check_cache.clear()
|
|
calls = 0
|
|
|
|
def provider() -> OperationalCheck:
|
|
nonlocal calls
|
|
calls += 1
|
|
return OperationalCheck("example.roundtrip", "Example", "ok", "Passed")
|
|
|
|
registration = OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.roundtrip",
|
|
provider=provider,
|
|
)
|
|
registry = _Registry(registration)
|
|
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert calls == 1
|
|
routes._module_operational_checks(registry, force=True)
|
|
assert calls == 2
|
|
|
|
|
|
def test_module_operational_check_failure_is_isolated() -> None:
|
|
routes._module_check_cache.clear()
|
|
|
|
def provider() -> OperationalCheck:
|
|
raise RuntimeError("secret detail")
|
|
|
|
result = routes._module_operational_checks(
|
|
_Registry(
|
|
OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.failed",
|
|
provider=provider,
|
|
)
|
|
),
|
|
force=True,
|
|
)[0]
|
|
|
|
assert result["state"] == "error"
|
|
assert "secret detail" not in result["detail"]
|
|
|
|
|
|
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [
|
|
{"role": "api", "state": "active", "stale": False},
|
|
{"role": "worker", "state": "active", "stale": True},
|
|
],
|
|
"expected": {"api": 2, "worker": 1},
|
|
"active": {"api": 1, "worker": 0},
|
|
"recovery": {"requires_attention": 1},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["recovery_required"] == 1
|
|
|
|
|
|
def test_shared_runtime_cluster_skew_or_unserved_queue_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [],
|
|
"expected": {"api": 2, "worker": 2},
|
|
"active": {"api": 2, "worker": 2},
|
|
"composition": {"skewed": True},
|
|
"software_versions": {"skewed": False},
|
|
"queues": {"missing": ["calendar"]},
|
|
"recovery": {"requires_attention": 0},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["composition_skewed"] is True
|
|
assert check["metrics"]["missing_queues"] == 1
|
|
|
|
|
|
def test_database_capacity_check_enforces_shared_rendered_budget(
|
|
monkeypatch,
|
|
) -> None:
|
|
monkeypatch.setattr(routes.core_settings, "state_profile", "shared")
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_limit", 100)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_reserve", 10)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_available", 90)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 70)
|
|
|
|
healthy = routes._database_capacity_check()
|
|
assert healthy["state"] == "ok"
|
|
assert healthy["metrics"]["peak"] == 70
|
|
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 91)
|
|
overrun = routes._database_capacity_check()
|
|
assert overrun["state"] == "error"
|
|
assert overrun["readiness_critical"] is True
|
|
|
|
|
|
def test_recovery_metrics_separate_failure_unknown_and_active_work() -> None:
|
|
metrics = routes._recovery_metrics(
|
|
[
|
|
{"status": "running"},
|
|
{"status": "failed"},
|
|
{"status": "outcome_unknown"},
|
|
{"status": "recovery_required"},
|
|
{"status": "manual_intervention"},
|
|
]
|
|
)
|
|
|
|
assert metrics == {
|
|
"failed": 2,
|
|
"outcome_unknown": 1,
|
|
"recovery_required": 1,
|
|
"active": 1,
|
|
"requires_attention": 4,
|
|
}
|
|
|
|
|
|
def test_local_storage_capacity_reports_bounded_filesystem_metrics(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
metrics = routes._local_storage_capacity(tmp_path)
|
|
|
|
assert metrics["backend"] == "local"
|
|
assert metrics["capacity_observable"] is True
|
|
assert metrics["capacity_total_bytes"] > 0
|
|
assert 0 <= metrics["capacity_used_percent"] <= 100
|