GovOPlaN/govoplan
2
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Enforce signed backup evidence before migrations

Browse Source
This commit is contained in:
zemion
2026-08-03 02:01:13 +02:00
parent 2c515f73c2
commit cbbe08d912
16 changed files with 2157 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/deployment/govoplan_deploy/cli.py
+291 -9
View File
@@ -20,7 +20,14 @@ from typing import Iterator, Mapping, Sequence
from urllib.error import URLError
from urllib.request import urlopen
from .backup_evidence import (
DEFAULT_MAX_BACKUP_AGE_SECONDS,
MAX_BACKUP_EVIDENCE_BYTES,
MAX_BACKUP_KEYRING_BYTES,
verify_backup_evidence,
)
from .bundle import (
BACKUP_RUNTIME_ENV_KEYS,
atomic_write,
bundle_paths,
canonical_json,
@@ -72,7 +79,12 @@ from .kubernetes import (
render_kubernetes,
write_secret_creation_hint,
)
from .planning import DeploymentPlan, build_plan
from .planning import (
DeploymentPlan,
build_plan,
release_change_requires_backup,
verify_stored_backup_evidence,
)
from .recovery import (
DeploymentOperationJournal,
list_operations,
@@ -183,6 +195,22 @@ def build_parser() -> argparse.ArgumentParser:
help="Load verified archives into Docker using fixed image-load commands.",
)
verify_backup = subparsers.add_parser(
"verify-backup",
help="Verify and optionally adopt signed coordinated backup evidence.",
)
_directory_argument(verify_backup)
evidence_source = verify_backup.add_mutually_exclusive_group(required=True)
evidence_source.add_argument("--evidence", type=Path)
evidence_source.add_argument("--evidence-url")
verify_backup.add_argument("--evidence-sha256", required=True)
verify_backup.add_argument("--trusted-keyring", type=Path, required=True)
verify_backup.add_argument(
"--allow-private-evidence-host",
action="store_true",
)
verify_backup.add_argument("--adopt", action="store_true")
kubernetes = subparsers.add_parser(
"render-kubernetes",
help="Export the stateless multi-host runtime for Kubernetes.",
@@ -392,6 +420,8 @@ def main(argv: Sequence[str] | None = None) -> int:
return _verify_release(args)
if args.command == "verify-offline-images":
return _verify_offline_images(args)
if args.command == "verify-backup":
return _verify_backup(args)
if args.command == "render-kubernetes":
return _render_kubernetes(args)
if args.command == "verify-kubernetes":
@@ -534,15 +564,17 @@ def _render_or_doctor(args: argparse.Namespace) -> int:
def _apply(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
spec = load_spec(paths.spec)
if args.allow_unverified_images and spec.profile != "evaluation":
raise ValueError(
"--allow-unverified-images is restricted to evaluation installations"
)
ensure_private_directory(paths.root)
with _deployment_lock(paths.lock):
spec = load_spec(paths.spec)
previous_receipt = _read_json_object(paths.receipt)
backup_required = release_change_requires_backup(spec, previous_receipt)
if args.allow_unverified_images and spec.profile != "evaluation":
raise ValueError(
"--allow-unverified-images is restricted to evaluation installations"
)
secrets = reconcile_runtime_environment(spec, read_env(paths.env))
_write_bundle(spec, paths, secrets)
secrets = _write_bundle(spec, paths, secrets)
plan = build_plan(spec, paths, include_host_checks=True)
_write_plan(paths.plan, plan)
effective_errors = [
@@ -577,6 +609,36 @@ def _apply(args: argparse.Namespace) -> int:
paths,
plan=plan.to_dict(),
)
try:
if backup_required:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=previous_receipt,
)
journal.record(
"backup-evidence-verified",
"succeeded",
dict(backup_summary),
)
else:
journal.record(
"backup-evidence-not-required",
"succeeded",
{"release_change": False},
)
except BaseException as exc:
journal.record(
"backup-evidence-rejected",
"blocked",
{
"phase": "preflight",
"exception_type": type(exc).__name__,
"migration_started": False,
},
)
journal.failed(exc)
raise
compose = [
docker,
"compose",
@@ -624,6 +686,29 @@ def _apply(args: argparse.Namespace) -> int:
"succeeded",
{"services": mutable_runtime_services, "timeout_seconds": 120},
)
if backup_required:
try:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=previous_receipt,
)
except BaseException as exc:
journal.record(
"backup-evidence-rejected",
"blocked",
{
"phase": "migration-boundary",
"exception_type": type(exc).__name__,
"migration_started": False,
},
)
raise
journal.record(
"backup-evidence-reverified",
"succeeded",
dict(backup_summary),
)
journal.migration_started()
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
journal.migration_completed()
@@ -898,6 +983,108 @@ def _verify_offline_images(args: argparse.Namespace) -> int:
return 0
def _verify_backup(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
ensure_private_directory(paths.root)
with _deployment_lock(paths.lock):
return _verify_backup_locked(args, paths)
def _verify_backup_locked(args: argparse.Namespace, paths) -> int:
spec = load_spec(paths.spec)
expected_digest = str(args.evidence_sha256 or "").strip().lower()
if len(expected_digest) != 64 or any(
character not in "0123456789abcdef" for character in expected_digest
):
raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest")
if args.evidence_url:
encoded_evidence = fetch_bounded_https(
str(args.evidence_url),
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
allow_private_host=args.allow_private_evidence_host,
)
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
else:
evidence_path = args.evidence.expanduser().resolve()
encoded_evidence = read_bounded_bytes(
evidence_path,
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
)
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
if encoded_evidence != canonical_distribution_json(evidence):
raise DistributionError("backup evidence is not canonical JSON")
if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest:
raise DistributionError("backup evidence SHA-256 does not match")
keyring_path = args.trusted_keyring.expanduser().resolve()
keyring = load_bounded_json(
keyring_path,
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
)
encoded_keyring = canonical_distribution_json(keyring)
receipt = _read_json_object(paths.receipt)
previous_release = receipt.get("release") if receipt else None
release: Mapping[str, object] = (
previous_release
if isinstance(previous_release, Mapping)
else {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"composition_sha256": spec.release.composition_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
}
)
summary = verify_backup_evidence(
evidence,
keyring,
installation_id=spec.installation_id,
profile=spec.profile,
release=release,
max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS,
)
print(
"Verified coordinated recovery point "
f"{summary['recovery_point_id']} with restore drill "
f"{summary['restore_drill_id']} and trusted key "
f"{summary['signature_key_id']}."
)
if not args.adopt:
return 0
verification = {
"schema_version": 1,
"evidence_sha256": expected_digest,
"keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(),
"signature_key_id": summary["signature_key_id"],
"verified_at": _now(),
"evidence_id": summary["evidence_id"],
"recovery_point_id": summary["recovery_point_id"],
"restore_drill_id": summary["restore_drill_id"],
"release_manifest_sha256": release.get("manifest_sha256"),
"captured_at": summary["captured_at"],
"expires_at": summary["expires_at"],
"restore_started_at": summary["restore_started_at"],
"restore_completed_at": summary["restore_completed_at"],
"measured_rpo_seconds": summary["measured_rpo_seconds"],
"measured_rto_seconds": summary["measured_rto_seconds"],
"component_count": summary["component_count"],
}
atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600)
atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600)
atomic_write(
paths.backup_verification,
canonical_json(verification),
mode=0o600,
)
_write_bundle(
spec,
paths,
reconcile_runtime_environment(spec, read_env(paths.env)),
)
print(f"Adopted signed backup evidence in {paths.root}.")
return 0
def _selected_dependency_images(
spec: InstallationSpec,
*,
@@ -929,6 +1116,30 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
spec = load_spec(paths.spec)
environment = reconcile_runtime_environment(spec, read_env(paths.env))
environment.update(_backup_runtime_environment(spec, paths))
receipt = _read_json_object(paths.receipt)
backup_required = release_change_requires_backup(spec, receipt)
backup_summary: Mapping[str, object] | None = None
evidence_files = (
paths.backup_evidence,
paths.backup_keyring,
paths.backup_verification,
)
if backup_required:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=receipt,
)
elif all(path.is_file() for path in evidence_files):
try:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=receipt,
)
except (DistributionError, OSError):
backup_summary = None
manifest = render_kubernetes(
spec,
environment,
@@ -936,6 +1147,8 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
secret_name=args.secret_name,
tls_secret_name=args.tls_secret_name,
ingress_class_name=args.ingress_class_name,
backup_required=backup_required,
backup_evidence=backup_summary,
)
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
atomic_write(output, canonical_json(manifest), mode=0o600)
@@ -1027,6 +1240,7 @@ def _deployment_receipt(
return {
"schema_version": 1,
"installation_id": spec.installation_id,
"profile": spec.profile,
"applied_at": _now(),
"spec_sha256": digest_json(spec.to_dict()),
"compose_sha256": digest_json(render_compose(spec)),
@@ -1064,6 +1278,16 @@ def _deployment_receipt(
}
def _read_json_object(path: Path) -> dict[str, object]:
if not path.is_file():
return {}
try:
value = load_bounded_json(path, maximum_bytes=64 * 1024)
except (DistributionError, OSError):
return {}
return value
def _updated_spec(
current: InstallationSpec, args: argparse.Namespace
) -> InstallationSpec:
@@ -1222,10 +1446,12 @@ def _write_bundle(
spec: InstallationSpec,
paths,
secrets: Mapping[str, str],
) -> None:
) -> dict[str, str]:
ensure_private_directory(paths.root)
runtime_environment = dict(secrets)
runtime_environment.update(_backup_runtime_environment(spec, paths))
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
write_env(paths.env, secrets)
write_env(paths.env, runtime_environment)
atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
atomic_write(
paths.load_balancer_config,
@@ -1247,6 +1473,62 @@ def _write_bundle(
render_garage_config().encode("utf-8"),
mode=0o644,
)
return dict(sorted(runtime_environment.items()))
def _backup_runtime_environment(
spec: InstallationSpec,
paths,
) -> dict[str, str]:
values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS}
evidence_files = (
paths.backup_evidence,
paths.backup_keyring,
paths.backup_verification,
)
if not any(path.is_file() for path in evidence_files):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent"
return values
if not all(path.is_file() for path in evidence_files):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
return values
verification = _read_json_object(paths.backup_verification)
try:
summary = verify_stored_backup_evidence(
spec,
paths,
receipt=_read_json_object(paths.receipt),
)
except (DistributionError, OSError):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
return values
values.update(
{
"GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
"GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]),
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]),
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]),
"GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]),
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str(
verification["release_manifest_sha256"]
),
"GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]),
"GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]),
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]),
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str(
summary["restore_completed_at"]
),
"GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]),
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str(
summary["measured_rpo_seconds"]
),
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str(
summary["measured_rto_seconds"]
),
"GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]),
}
)
return values
def _write_plan(path: Path, plan: DeploymentPlan) -> None: