GovOPlaN/govoplan-ops
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Expose sanitized backup restore evidence

Browse Source
This commit is contained in:
zemion
2026-08-03 02:02:36 +02:00
parent 1426fd96b1
commit 0105fd49f5
5 changed files with 301 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_ops/backend/api/v1/routes.py
+148 -8
View File
@@ -5,6 +5,7 @@ import shutil
import threading
import time
from collections.abc import Mapping
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
@@ -259,9 +260,7 @@ def _ops_status_payload(
"requires_attention", 0
),
"failed_operation_count": int(recovery_metrics.get("failed") or 0),
"outcome_unknown_count": int(
recovery_metrics.get("outcome_unknown") or 0
),
"outcome_unknown_count": int(recovery_metrics.get("outcome_unknown") or 0),
"active_operation_count": int(recovery_metrics.get("active") or 0),
},
"readiness": readiness,
@@ -408,8 +407,7 @@ def _recovery_metrics(operations: list[dict[str, Any]]) -> dict[str, int]:
value in {"recovery_required", "recovering"} for value in statuses
)
active = sum(
value in {"planned", "prepared", "running", "recovering"}
for value in statuses
value in {"planned", "prepared", "running", "recovering"} for value in statuses
)
return {
"failed": failed,
@@ -906,9 +904,7 @@ def _local_storage_capacity(root: Path) -> dict[str, Any]:
usage = shutil.disk_usage(root)
except OSError:
return {"backend": "local", "capacity_observable": False}
used_percent = (
round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
)
used_percent = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
return {
"backend": "local",
"capacity_observable": True,
@@ -987,6 +983,150 @@ def _cached_module_check(
def _backup_restore_check() -> dict[str, Any]:
projected = _deployer_backup_restore_check()
if projected is not None:
return projected
return _legacy_backup_restore_check()
def _deployer_backup_restore_check() -> dict[str, Any] | None:
state = str(os.environ.get("GOVOPLAN_BACKUP_EVIDENCE_STATE") or "").strip()
if not state:
return None
if state == "absent":
return _check(
"backup_restore_evidence",
"Backup and restore evidence",
"warning",
"No signed coordinated backup and isolated-restore evidence is adopted.",
metrics={"evidence_state": "absent", "restore_drill_ok": False},
)
if state != "verified":
return _check(
"backup_restore_evidence",
"Backup and restore evidence",
"warning",
"Deployment backup evidence is incomplete, stale, or failed verification.",
metrics={"evidence_state": "invalid", "restore_drill_ok": False},
)
values = {
"evidence_id": os.environ.get("GOVOPLAN_BACKUP_EVIDENCE_ID", ""),
"recovery_point_id": os.environ.get("GOVOPLAN_BACKUP_RECOVERY_POINT_ID", ""),
"restore_drill_id": os.environ.get("GOVOPLAN_BACKUP_RESTORE_DRILL_ID", ""),
"evidence_sha256": os.environ.get("GOVOPLAN_BACKUP_EVIDENCE_SHA256", ""),
"release_manifest_sha256": os.environ.get(
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256", ""
),
"captured_at": os.environ.get("GOVOPLAN_BACKUP_CAPTURED_AT", ""),
"expires_at": os.environ.get("GOVOPLAN_BACKUP_EXPIRES_AT", ""),
"restore_started_at": os.environ.get("GOVOPLAN_BACKUP_RESTORE_STARTED_AT", ""),
"restore_completed_at": os.environ.get(
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT", ""
),
"verified_at": os.environ.get("GOVOPLAN_BACKUP_VERIFIED_AT", ""),
"measured_rpo_seconds": os.environ.get(
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS", ""
),
"measured_rto_seconds": os.environ.get(
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS", ""
),
"component_count": os.environ.get("GOVOPLAN_BACKUP_COMPONENT_COUNT", ""),
}
try:
expires_at = _ops_timestamp(values["expires_at"])
captured_at = _ops_timestamp(values["captured_at"])
restore_started_at = _ops_timestamp(values["restore_started_at"])
restore_completed_at = _ops_timestamp(values["restore_completed_at"])
verified_at = _ops_timestamp(values["verified_at"])
measured_rpo = _ops_nonnegative_integer(values["measured_rpo_seconds"])
measured_rto = _ops_nonnegative_integer(values["measured_rto_seconds"])
component_count = _ops_nonnegative_integer(values["component_count"])
if component_count < 4:
raise ValueError("partial component set")
if any(
len(values[field]) != 64
or any(character not in "0123456789abcdef" for character in values[field])
for field in ("evidence_sha256", "release_manifest_sha256")
):
raise ValueError("invalid digest")
if not all(
values[field]
for field in ("evidence_id", "recovery_point_id", "restore_drill_id")
):
raise ValueError("missing identity")
if not captured_at <= restore_started_at <= restore_completed_at <= verified_at:
raise ValueError("invalid chronology")
if (
abs(
(restore_completed_at - restore_started_at).total_seconds()
- measured_rto
)
> 5
):
raise ValueError("RTO does not match chronology")
except ValueError:
return _check(
"backup_restore_evidence",
"Backup and restore evidence",
"warning",
"The sanitized deployment backup receipt is malformed.",
metrics={"evidence_state": "invalid", "restore_drill_ok": False},
)
expired = expires_at <= datetime.now(UTC)
return _check(
"backup_restore_evidence",
"Backup and restore evidence",
"warning" if expired else "ok",
(
"Signed backup and restore evidence has expired; adopt a fresh recovery point before migration."
if expired
else (
f"Recovery point {values['recovery_point_id']} and isolated restore "
f"drill {values['restore_drill_id']} are deployment-verified."
)
),
metrics={
"evidence_state": "expired" if expired else "verified",
"evidence_id": values["evidence_id"],
"recovery_point_id": values["recovery_point_id"],
"restore_drill_id": values["restore_drill_id"],
"captured_at": values["captured_at"],
"expires_at": values["expires_at"],
"restore_started_at": values["restore_started_at"],
"restore_completed_at": values["restore_completed_at"],
"measured_rpo_seconds": measured_rpo,
"measured_rto_seconds": measured_rto,
"component_count": component_count,
"restore_drill_ok": not expired,
},
)
def _ops_timestamp(value: str) -> datetime:
if not value or len(value) > 64:
raise ValueError("invalid timestamp")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError("invalid timestamp") from exc
if parsed.tzinfo is None:
raise ValueError("timestamp has no timezone")
return parsed.astimezone(UTC)
def _ops_nonnegative_integer(value: str) -> int:
if not value.isascii() or not value.isdigit() or len(value) > 16:
raise ValueError("invalid integer")
parsed = int(value)
if parsed > 2**63 - 1:
raise ValueError("integer out of bounds")
return parsed
def _legacy_backup_restore_check() -> dict[str, Any]:
runtime_dir = default_installer_runtime_dir(core_settings.database_url)
runs = list_module_installer_runs(runtime_dir=runtime_dir, limit=25)
latest_evidence: tuple[dict[str, object], Mapping[str, object]] | None = None
src/govoplan_ops/backend/manifest.py
+74 -12
View File
@@ -1,8 +1,25 @@
from __future__ import annotations
from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER
from govoplan_core.core.modules import DocumentationCondition, DocumentationTopic, FrontendModule, FrontendRoute, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate
from govoplan_core.core.provider_governance import ModuleArchitectureDeclaration, ModuleArchitectureDocumentation, ModuleMaturityEvidence
from govoplan_core.core.access import (
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
)
from govoplan_core.core.modules import (
DocumentationCondition,
DocumentationTopic,
FrontendModule,
FrontendRoute,
ModuleContext,
ModuleManifest,
NavItem,
PermissionDefinition,
RoleTemplate,
)
from govoplan_core.core.provider_governance import (
ModuleArchitectureDeclaration,
ModuleArchitectureDocumentation,
ModuleMaturityEvidence,
)
from govoplan_core.core.views import ViewSurface
OPS_READ_SCOPE = "ops:operations:read"
@@ -25,6 +42,11 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
reference="docs/SCALABILITY_PROFILES.md",
summary="Documents operational topology and scaling posture.",
),
ModuleMaturityEvidence(
kind="documentation",
reference="docs/BACKUP_EVIDENCE_STATUS.md",
summary="Documents the sanitized signed backup and restore status projection.",
),
),
known_limits=(
"Provider health observations depend on module-owned operational probes and may be unavailable until configured.",
@@ -32,8 +54,14 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
owned_concepts=("operations status projection", "bounded operational probes"),
non_owned_concepts=("domain repair", "external provider credentials"),
documentation=ModuleArchitectureDocumentation(
recovery=("docs/SCALABILITY_PROFILES.md",),
operations=("docs/SCALABILITY_PROFILES.md",),
recovery=(
"docs/SCALABILITY_PROFILES.md",
"docs/BACKUP_EVIDENCE_STATUS.md",
),
operations=(
"docs/SCALABILITY_PROFILES.md",
"docs/BACKUP_EVIDENCE_STATUS.md",
),
),
)
@@ -63,7 +91,10 @@ manifest = ModuleManifest(
id="ops",
name="Ops",
version="0.1.8",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
required_capabilities=(
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
),
optional_dependencies=("audit", "docs", "notifications"),
permissions=(
_permission(
@@ -108,7 +139,7 @@ manifest = ModuleManifest(
id="ops.runtime-coordination-and-recovery",
title="Drain runtime nodes and inspect recovery evidence",
summary="Ops projects shared runtime heartbeats, replica gaps, drain controls, and recovery states that require operator attention.",
body="Use the runtime table to identify stale or composition-skewed API and worker replicas. Drain before replacement so API readiness closes and workers stop taking new queue work; cancellation is available while the node is still draining. The recovery table reports durable Core recovery operations, but a recorded forward-recovery or manual-intervention state is not an automatic database restore.",
body="Use the runtime table to identify stale or composition-skewed API and worker replicas. Drain before replacement so API readiness closes and workers stop taking new queue work; cancellation is available while the node is still draining. The recovery table reports durable Core recovery operations. Backup status separately projects only the sanitized deployment verification receipt: a verified status identifies a coordinated recovery point and isolated restore drill, while absent, expired, or invalid evidence blocks a release-changing migration.",
documentation_types=("admin", "user"),
audience=("operator", "system_admin"),
conditions=(
@@ -126,24 +157,55 @@ manifest = ModuleManifest(
"Compare active non-stale nodes with the configured API and worker replica expectations.",
"Request drain and wait for the node to report draining before replacing it.",
"Inspect every recovery-required, outcome-unknown, or manual-intervention operation and follow its recorded recovery mode.",
"Confirm that backup evidence is verified and current before authorizing a release-changing migration.",
"Verify replacement composition, readiness, queue consumers, and recovery evidence before closing the operation.",
],
"limitations": [
"Drain is observed on the runtime heartbeat interval and does not forcibly terminate active work.",
"Ops does not create database backups or make an unsafe post-migration rollback reversible.",
"Ops does not create or restore backups and never receives private artifact or key-custody references.",
"A verified receipt proves the recorded drill; it does not make an unsafe post-migration code rollback reversible.",
],
},
),
),
route_factory=_route_factory,
nav_items=(NavItem(path="/ops", label="Ops", icon="activity", required_any=OPS_READ_SCOPES, order=890),),
nav_items=(
NavItem(
path="/ops",
label="Ops",
icon="activity",
required_any=OPS_READ_SCOPES,
order=890,
),
),
frontend=FrontendModule(
module_id="ops",
package_name="@govoplan/ops-webui",
routes=(FrontendRoute(path="/ops", component="OpsPage", required_any=OPS_READ_SCOPES, order=890),),
nav_items=(NavItem(path="/ops", label="Ops", icon="activity", required_any=OPS_READ_SCOPES, order=890),),
routes=(
FrontendRoute(
path="/ops",
component="OpsPage",
required_any=OPS_READ_SCOPES,
order=890,
),
),
nav_items=(
NavItem(
path="/ops",
label="Ops",
icon="activity",
required_any=OPS_READ_SCOPES,
order=890,
),
),
view_surfaces=(
ViewSurface(id="ops.widget.health", module_id="ops", kind="section", label="Operations health widget", order=100),
ViewSurface(
id="ops.widget.health",
module_id="ops",
kind="section",
label="Operations health widget",
order=100,
),
),
),
architecture=ARCHITECTURE,