310 lines
10 KiB
Python
310 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from govoplan_core.core.operations import (
|
|
OperationalCheck,
|
|
OperationalCheckProviderRegistration,
|
|
RuntimeWorkStatus,
|
|
RuntimeWorkStatusContext,
|
|
RuntimeWorkStatusProviderRegistration,
|
|
)
|
|
from govoplan_ops.backend.api.v1 import routes
|
|
|
|
|
|
@dataclass
|
|
class _Manifest:
|
|
operational_check_providers: tuple[OperationalCheckProviderRegistration, ...]
|
|
runtime_work_status_providers: tuple[RuntimeWorkStatusProviderRegistration, ...] = ()
|
|
|
|
|
|
class _Registry:
|
|
def __init__(self, *registrations: OperationalCheckProviderRegistration):
|
|
self._manifest = _Manifest(tuple(registrations))
|
|
|
|
def manifests(self):
|
|
return (self._manifest,)
|
|
|
|
|
|
def test_module_operational_checks_cache_and_force() -> None:
|
|
routes._module_check_cache.clear()
|
|
calls = 0
|
|
|
|
def provider() -> OperationalCheck:
|
|
nonlocal calls
|
|
calls += 1
|
|
return OperationalCheck("example.roundtrip", "Example", "ok", "Passed")
|
|
|
|
registration = OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.roundtrip",
|
|
provider=provider,
|
|
)
|
|
registry = _Registry(registration)
|
|
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert routes._module_operational_checks(registry, force=False)[0]["state"] == "ok"
|
|
assert calls == 1
|
|
routes._module_operational_checks(registry, force=True)
|
|
assert calls == 2
|
|
|
|
|
|
def test_module_operational_check_failure_is_isolated() -> None:
|
|
routes._module_check_cache.clear()
|
|
|
|
def provider() -> OperationalCheck:
|
|
raise RuntimeError("secret detail")
|
|
|
|
result = routes._module_operational_checks(
|
|
_Registry(
|
|
OperationalCheckProviderRegistration(
|
|
module_id="example",
|
|
check_id="example.failed",
|
|
provider=provider,
|
|
)
|
|
),
|
|
force=True,
|
|
)[0]
|
|
|
|
assert result["state"] == "error"
|
|
assert "secret detail" not in result["detail"]
|
|
|
|
|
|
def test_runtime_work_provider_cache_force_and_unknown_metrics() -> None:
|
|
routes._runtime_work_cache.clear()
|
|
calls = 0
|
|
|
|
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
|
nonlocal calls
|
|
calls += 1
|
|
return RuntimeWorkStatus(
|
|
provider_id="example.queue",
|
|
label="Example queue",
|
|
backend="Example",
|
|
enabled=True,
|
|
configured=True,
|
|
state="healthy",
|
|
detail="Workers answered; queue depth unsupported.",
|
|
observed_at=context.observed_at,
|
|
active_workers=1,
|
|
queue_depths={"example": None},
|
|
)
|
|
|
|
registry = _Registry()
|
|
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
|
RuntimeWorkStatusProviderRegistration(
|
|
module_id="example",
|
|
provider_id="example.queue",
|
|
provider=provider,
|
|
),
|
|
)
|
|
context = RuntimeWorkStatusContext(
|
|
profile="split-worker",
|
|
observed_at=datetime.now(UTC),
|
|
stale_after_seconds=60,
|
|
)
|
|
|
|
first = routes._runtime_work_statuses(registry, context)
|
|
second = routes._runtime_work_statuses(registry, context)
|
|
forced = routes._runtime_work_statuses(registry, context, force=True)
|
|
|
|
assert first[0]["queue_depths"] == {"example": None}
|
|
assert second == first
|
|
assert forced[0]["state"] == "healthy"
|
|
assert calls == 2
|
|
|
|
|
|
def test_runtime_work_provider_failure_is_sanitized() -> None:
|
|
routes._runtime_work_cache.clear()
|
|
|
|
def provider(context: RuntimeWorkStatusContext) -> RuntimeWorkStatus:
|
|
del context
|
|
raise RuntimeError("redis://user:secret@example.test")
|
|
|
|
registry = _Registry()
|
|
registry._manifest.runtime_work_status_providers = ( # type: ignore[misc]
|
|
RuntimeWorkStatusProviderRegistration(
|
|
module_id="example",
|
|
provider_id="example.failed",
|
|
provider=provider,
|
|
),
|
|
)
|
|
result = routes._runtime_work_statuses(
|
|
registry,
|
|
RuntimeWorkStatusContext(
|
|
profile="split-worker",
|
|
observed_at=datetime.now(UTC),
|
|
stale_after_seconds=60,
|
|
),
|
|
force=True,
|
|
)[0]
|
|
|
|
assert result["state"] == "unreachable"
|
|
assert "secret" not in result["detail"]
|
|
|
|
|
|
def test_disabled_workers_are_expected_only_in_development() -> None:
|
|
disabled = {
|
|
"provider_id": "example.queue",
|
|
"state": "disabled",
|
|
"detail": "Intentionally disabled.",
|
|
"enabled": False,
|
|
"configured": True,
|
|
"queue_depths": {},
|
|
}
|
|
|
|
development = routes._runtime_work_check([disabled], "local-dev")
|
|
production = routes._runtime_work_check([disabled], "single-process")
|
|
|
|
assert development["state"] == "inactive"
|
|
assert development["readiness_critical"] is False
|
|
assert production["state"] == "warning"
|
|
assert production["readiness_critical"] is True
|
|
|
|
|
|
def test_shared_runtime_cluster_missing_replicas_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [
|
|
{"role": "api", "state": "active", "stale": False},
|
|
{"role": "worker", "state": "active", "stale": True},
|
|
],
|
|
"expected": {"api": 2, "worker": 1},
|
|
"active": {"api": 1, "worker": 0},
|
|
"recovery": {"requires_attention": 1},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["recovery_required"] == 1
|
|
|
|
|
|
def test_shared_runtime_cluster_skew_or_unserved_queue_blocks_readiness() -> None:
|
|
check = routes._runtime_cluster_check(
|
|
{
|
|
"available": True,
|
|
"state_profile": "shared",
|
|
"nodes": [],
|
|
"expected": {"api": 2, "worker": 2},
|
|
"active": {"api": 2, "worker": 2},
|
|
"composition": {"skewed": True},
|
|
"software_versions": {"skewed": False},
|
|
"queues": {"missing": ["calendar"]},
|
|
"recovery": {"requires_attention": 0},
|
|
}
|
|
)
|
|
|
|
assert check["state"] == "error"
|
|
assert check["readiness_critical"] is True
|
|
assert check["metrics"]["composition_skewed"] is True
|
|
assert check["metrics"]["missing_queues"] == 1
|
|
|
|
|
|
def test_database_capacity_check_enforces_shared_rendered_budget(
|
|
monkeypatch,
|
|
) -> None:
|
|
monkeypatch.setattr(routes.core_settings, "state_profile", "shared")
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_limit", 100)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_reserve", 10)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_available", 90)
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 70)
|
|
|
|
healthy = routes._database_capacity_check()
|
|
assert healthy["state"] == "ok"
|
|
assert healthy["metrics"]["peak"] == 70
|
|
|
|
monkeypatch.setattr(routes.core_settings, "database_connection_peak", 91)
|
|
overrun = routes._database_capacity_check()
|
|
assert overrun["state"] == "error"
|
|
assert overrun["readiness_critical"] is True
|
|
|
|
|
|
def test_recovery_metrics_separate_failure_unknown_and_active_work() -> None:
|
|
metrics = routes._recovery_metrics(
|
|
[
|
|
{"status": "running"},
|
|
{"status": "failed"},
|
|
{"status": "outcome_unknown"},
|
|
{"status": "recovery_required"},
|
|
{"status": "manual_intervention"},
|
|
]
|
|
)
|
|
|
|
assert metrics == {
|
|
"failed": 2,
|
|
"outcome_unknown": 1,
|
|
"recovery_required": 1,
|
|
"active": 1,
|
|
"requires_attention": 4,
|
|
}
|
|
|
|
|
|
def test_local_storage_capacity_reports_bounded_filesystem_metrics(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
metrics = routes._local_storage_capacity(tmp_path)
|
|
|
|
assert metrics["backend"] == "local"
|
|
assert metrics["capacity_observable"] is True
|
|
assert metrics["capacity_total_bytes"] > 0
|
|
assert 0 <= metrics["capacity_used_percent"] <= 100
|
|
|
|
|
|
def test_deployer_backup_receipt_reports_verified_and_expired_state(
|
|
monkeypatch,
|
|
) -> None:
|
|
now = datetime.now(UTC)
|
|
values = {
|
|
"GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
|
|
"GOVOPLAN_BACKUP_EVIDENCE_ID": "evidence-1",
|
|
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID": "recovery-1",
|
|
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID": "drill-1",
|
|
"GOVOPLAN_BACKUP_EVIDENCE_SHA256": "a" * 64,
|
|
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": "b" * 64,
|
|
"GOVOPLAN_BACKUP_CAPTURED_AT": (now - timedelta(hours=2)).isoformat(),
|
|
"GOVOPLAN_BACKUP_EXPIRES_AT": (now + timedelta(hours=2)).isoformat(),
|
|
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT": (
|
|
now - timedelta(hours=1, minutes=5)
|
|
).isoformat(),
|
|
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": (now - timedelta(hours=1)).isoformat(),
|
|
"GOVOPLAN_BACKUP_VERIFIED_AT": (now - timedelta(minutes=30)).isoformat(),
|
|
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": "120",
|
|
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": "300",
|
|
"GOVOPLAN_BACKUP_COMPONENT_COUNT": "4",
|
|
}
|
|
for key, value in values.items():
|
|
monkeypatch.setenv(key, value)
|
|
|
|
verified = routes._backup_restore_check()
|
|
assert verified["state"] == "ok"
|
|
assert verified["metrics"]["recovery_point_id"] == "recovery-1"
|
|
assert verified["metrics"]["measured_rto_seconds"] == 300
|
|
|
|
monkeypatch.setenv(
|
|
"GOVOPLAN_BACKUP_EXPIRES_AT",
|
|
(now - timedelta(minutes=1)).isoformat(),
|
|
)
|
|
expired = routes._backup_restore_check()
|
|
assert expired["state"] == "warning"
|
|
assert expired["metrics"]["evidence_state"] == "expired"
|
|
|
|
|
|
def test_deployer_backup_receipt_fails_closed_without_exposing_private_evidence(
|
|
monkeypatch,
|
|
) -> None:
|
|
monkeypatch.setenv("GOVOPLAN_BACKUP_EVIDENCE_STATE", "invalid")
|
|
invalid = routes._backup_restore_check()
|
|
|
|
assert invalid["state"] == "warning"
|
|
assert invalid["metrics"] == {
|
|
"evidence_state": "invalid",
|
|
"restore_drill_ok": False,
|
|
}
|
|
assert "artifact" not in invalid["detail"].lower()
|
|
assert "key" not in invalid["detail"].lower()
|