Enforce signed backup evidence before migrations

This commit is contained in:
2026-08-03 02:01:13 +02:00
parent 2c515f73c2
commit cbbe08d912
16 changed files with 2157 additions and 52 deletions
@@ -0,0 +1,500 @@
"""Signed, provider-neutral backup and isolated-restore evidence."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
import re
from typing import Any, Mapping
from urllib.parse import urlsplit
from .distribution import (
DistributionError,
load_bounded_json,
verify_signed_document,
)
MAX_BACKUP_EVIDENCE_BYTES = 1024 * 1024
MAX_BACKUP_KEYRING_BYTES = 1024 * 1024
DEFAULT_MAX_BACKUP_AGE_SECONDS = 24 * 60 * 60
MAX_COORDINATION_SKEW_SECONDS = 5 * 60
SHA256 = re.compile(r"^[0-9a-f]{64}$")
TOKEN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
REFERENCE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:[^\s]{1,2040}$")
def load_backup_evidence(path: Path) -> dict[str, Any]:
return load_bounded_json(path, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES)
def load_backup_keyring(path: Path) -> dict[str, Any]:
return load_bounded_json(path, maximum_bytes=MAX_BACKUP_KEYRING_BYTES)
def verify_backup_evidence(
payload: Mapping[str, Any],
keyring: Mapping[str, Any],
*,
installation_id: str,
profile: str,
release: Mapping[str, object],
now: datetime | None = None,
max_age_seconds: int = DEFAULT_MAX_BACKUP_AGE_SECONDS,
openssl: str = "openssl",
) -> dict[str, object]:
current = (now or datetime.now(UTC)).astimezone(UTC)
values = _validate_payload(payload, now=current, max_age_seconds=max_age_seconds)
if payload.get("installation_id") != installation_id:
raise DistributionError("backup evidence belongs to another installation")
subject = _object(payload.get("deployment_subject"), "deployment_subject")
if subject.get("profile") != profile:
raise DistributionError("backup evidence belongs to another deployment profile")
evidence_release = _object(payload.get("release"), "release")
for field in (
"channel",
"version",
"manifest_sha256",
"composition_sha256",
"api_image",
"web_image",
):
if evidence_release.get(field) != release.get(field):
raise DistributionError(
f"backup evidence does not match release field {field!r}"
)
key_id = verify_signed_document(
payload,
keyring,
purpose="govoplan-backup-evidence",
label="backup evidence",
now=current,
openssl=openssl,
)
recovery_point = _object(payload.get("recovery_point"), "recovery_point")
restore = _object(payload.get("restore_drill"), "restore_drill")
return {
"evidence_id": payload["evidence_id"],
"recovery_point_id": recovery_point["id"],
"captured_at": recovery_point["captured_at"],
"expires_at": payload["expires_at"],
"restore_drill_id": restore["drill_id"],
"restore_started_at": restore["started_at"],
"restore_completed_at": restore["completed_at"],
"measured_rpo_seconds": restore["measured_rpo_seconds"],
"measured_rto_seconds": restore["measured_rto_seconds"],
"signature_key_id": key_id,
"component_count": values["component_count"],
}
def _validate_payload(
payload: Mapping[str, Any],
*,
now: datetime,
max_age_seconds: int,
) -> dict[str, int]:
if max_age_seconds < 60 or max_age_seconds > 30 * 24 * 60 * 60:
raise DistributionError("backup maximum age is out of bounds")
_exact_keys(
payload,
{
"schema_version",
"evidence_id",
"installation_id",
"deployment_subject",
"release",
"recovery_point",
"components",
"restore_drill",
"issued_at",
"expires_at",
"revoked",
"signatures",
},
"backup evidence",
)
if payload.get("schema_version") != "1":
raise DistributionError("unsupported backup evidence schema_version")
_token(payload.get("evidence_id"), "evidence_id")
_token(payload.get("installation_id"), "installation_id")
issued = _timestamp(payload.get("issued_at"), "issued_at")
expires = _timestamp(payload.get("expires_at"), "expires_at")
if issued > now or expires <= issued or expires <= now:
raise DistributionError("backup evidence is not currently valid")
if payload.get("revoked") is not False:
raise DistributionError("backup evidence is revoked")
subject = _object(payload.get("deployment_subject"), "deployment_subject")
_exact_keys(subject, {"profile", "topology", "subject_ref"}, "deployment_subject")
if subject.get("profile") not in {"evaluation", "self-hosted"}:
raise DistributionError("deployment_subject.profile is invalid")
_token(subject.get("topology"), "deployment_subject.topology")
_reference(subject.get("subject_ref"), "deployment_subject.subject_ref")
release = _object(payload.get("release"), "release")
_exact_keys(
release,
{
"channel",
"version",
"manifest_sha256",
"composition_sha256",
"api_image",
"web_image",
},
"release",
)
_token(release.get("channel"), "release.channel")
_token(release.get("version"), "release.version")
_sha256(release.get("manifest_sha256"), "release.manifest_sha256")
_sha256(release.get("composition_sha256"), "release.composition_sha256")
_image(release.get("api_image"), "release.api_image")
_image(release.get("web_image"), "release.web_image")
recovery = _object(payload.get("recovery_point"), "recovery_point")
_exact_keys(
recovery,
{"id", "captured_at", "consistency", "rpo_seconds", "write_fence"},
"recovery_point",
)
recovery_id = _token(recovery.get("id"), "recovery_point.id")
captured = _timestamp(recovery.get("captured_at"), "recovery_point.captured_at")
age = (now - captured).total_seconds()
if age < 0 or age > max_age_seconds:
raise DistributionError("backup recovery point is stale or in the future")
if recovery.get("consistency") not in {
"provider-atomic",
"application-quiesced",
"transaction-consistent",
}:
raise DistributionError("recovery_point.consistency is invalid")
declared_rpo = _bounded_integer(
recovery.get("rpo_seconds"),
"recovery_point.rpo_seconds",
maximum=30 * 24 * 60 * 60,
)
fence = _object(recovery.get("write_fence"), "recovery_point.write_fence")
_exact_keys(
fence,
{"mode", "token_sha256", "established_at"},
"recovery_point.write_fence",
)
if fence.get("mode") not in {
"provider-snapshot",
"application-quiesce",
"transaction-boundary",
}:
raise DistributionError("recovery_point.write_fence.mode is invalid")
_sha256(fence.get("token_sha256"), "recovery_point.write_fence.token_sha256")
established = _timestamp(
fence.get("established_at"),
"recovery_point.write_fence.established_at",
)
if abs((captured - established).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS:
raise DistributionError(
"backup write fence is not coordinated with recovery point"
)
components = _object(payload.get("components"), "components")
_exact_keys(
components,
{"database", "objects", "configuration", "key_custody"},
"components",
)
captured_components = [
_database_component(components.get("database")),
_objects_component(components.get("objects")),
_configuration_component(components.get("configuration")),
_key_custody_component(components.get("key_custody")),
]
if any(
abs((component_time - captured).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS
for component_time in captured_components
):
raise DistributionError("backup components do not share one recovery point")
restore = _object(payload.get("restore_drill"), "restore_drill")
_exact_keys(
restore,
{
"drill_id",
"recovery_point_id",
"started_at",
"completed_at",
"isolated_target_ref",
"release_manifest_sha256",
"migration_heads_sha256",
"representative_object_manifest_sha256",
"database_verified",
"objects_verified",
"configuration_verified",
"key_custody_verified",
"semantic_checks",
"measured_rpo_seconds",
"measured_rto_seconds",
"evidence_ref",
},
"restore_drill",
)
_token(restore.get("drill_id"), "restore_drill.drill_id")
if restore.get("recovery_point_id") != recovery_id:
raise DistributionError("restore drill used another recovery point")
started = _timestamp(restore.get("started_at"), "restore_drill.started_at")
completed = _timestamp(restore.get("completed_at"), "restore_drill.completed_at")
if started < captured or completed < started or completed > issued:
raise DistributionError(
"restore drill completion is outside evidence chronology"
)
_reference(restore.get("isolated_target_ref"), "restore_drill.isolated_target_ref")
_reference(restore.get("evidence_ref"), "restore_drill.evidence_ref")
for field in (
"release_manifest_sha256",
"migration_heads_sha256",
"representative_object_manifest_sha256",
):
_sha256(restore.get(field), f"restore_drill.{field}")
if restore.get("release_manifest_sha256") != release.get("manifest_sha256"):
raise DistributionError("restore drill used another immutable release")
for field in (
"database_verified",
"objects_verified",
"configuration_verified",
"key_custody_verified",
):
if restore.get(field) is not True:
raise DistributionError(f"restore_drill.{field} must be true")
semantic = restore.get("semantic_checks")
if not isinstance(semantic, list) or not semantic or len(semantic) > 128:
raise DistributionError("restore_drill.semantic_checks must not be empty")
seen_checks: set[str] = set()
for index, raw in enumerate(semantic):
check = _object(raw, f"restore_drill.semantic_checks[{index}]")
_exact_keys(
check,
{"id", "status", "evidence_ref"},
f"restore_drill.semantic_checks[{index}]",
)
check_id = _token(check.get("id"), f"semantic_checks[{index}].id")
if check_id in seen_checks or check.get("status") != "passed":
raise DistributionError("restore drill semantic checks are invalid")
seen_checks.add(check_id)
_reference(check.get("evidence_ref"), f"semantic_checks[{index}].evidence_ref")
measured_rpo = _bounded_integer(
restore.get("measured_rpo_seconds"),
"restore_drill.measured_rpo_seconds",
maximum=30 * 24 * 60 * 60,
)
measured_rto = _bounded_integer(
restore.get("measured_rto_seconds"),
"restore_drill.measured_rto_seconds",
maximum=30 * 24 * 60 * 60,
)
if abs((completed - started).total_seconds() - measured_rto) > 5:
raise DistributionError("restore drill RTO does not match its timestamps")
if measured_rpo > declared_rpo:
raise DistributionError(
"restore drill exceeds the declared recovery point objective"
)
_validate_signatures(payload.get("signatures"))
return {"component_count": len(captured_components)}
def _database_component(raw: object) -> datetime:
value = _object(raw, "components.database")
_exact_keys(
value,
{
"provider",
"artifact_ref",
"artifact_sha256",
"snapshot_id",
"lsn",
"protected",
"encryption_key_ref",
"captured_at",
},
"components.database",
)
_common_artifact(value, "components.database")
_token(value.get("snapshot_id"), "components.database.snapshot_id")
_bounded_text(value.get("lsn"), "components.database.lsn", maximum=256)
return _timestamp(value.get("captured_at"), "components.database.captured_at")
def _objects_component(raw: object) -> datetime:
value = _object(raw, "components.objects")
_exact_keys(
value,
{
"provider",
"artifact_ref",
"manifest_sha256",
"version_id",
"object_count",
"total_bytes",
"protected",
"encryption_key_ref",
"captured_at",
},
"components.objects",
)
_token(value.get("provider"), "components.objects.provider")
_reference(value.get("artifact_ref"), "components.objects.artifact_ref")
_sha256(value.get("manifest_sha256"), "components.objects.manifest_sha256")
_token(value.get("version_id"), "components.objects.version_id")
_bounded_integer(value.get("object_count"), "components.objects.object_count")
_bounded_integer(value.get("total_bytes"), "components.objects.total_bytes")
_protected_key_reference(value, "components.objects")
return _timestamp(value.get("captured_at"), "components.objects.captured_at")
def _configuration_component(raw: object) -> datetime:
value = _object(raw, "components.configuration")
_exact_keys(
value,
{
"artifact_ref",
"sha256",
"protected",
"encryption_key_ref",
"captured_at",
},
"components.configuration",
)
_reference(value.get("artifact_ref"), "components.configuration.artifact_ref")
_sha256(value.get("sha256"), "components.configuration.sha256")
_protected_key_reference(value, "components.configuration")
return _timestamp(value.get("captured_at"), "components.configuration.captured_at")
def _key_custody_component(raw: object) -> datetime:
value = _object(raw, "components.key_custody")
_exact_keys(
value,
{"provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"},
"components.key_custody",
)
_token(value.get("provider"), "components.key_custody.provider")
_reference(value.get("keyset_ref"), "components.key_custody.keyset_ref")
_token(value.get("keyset_version"), "components.key_custody.keyset_version")
if value.get("recoverable") is not True:
raise DistributionError("components.key_custody.recoverable must be true")
return _timestamp(value.get("captured_at"), "components.key_custody.captured_at")
def _common_artifact(value: Mapping[str, Any], label: str) -> None:
_token(value.get("provider"), f"{label}.provider")
_reference(value.get("artifact_ref"), f"{label}.artifact_ref")
_sha256(value.get("artifact_sha256"), f"{label}.artifact_sha256")
_protected_key_reference(value, label)
def _protected_key_reference(value: Mapping[str, Any], label: str) -> None:
if value.get("protected") is not True:
raise DistributionError(f"{label}.protected must be true")
_reference(value.get("encryption_key_ref"), f"{label}.encryption_key_ref")
def _validate_signatures(raw: object) -> None:
if not isinstance(raw, list) or not raw or len(raw) > 16:
raise DistributionError("backup evidence signatures must not be empty")
seen: set[str] = set()
for index, item in enumerate(raw):
signature = _object(item, f"signatures[{index}]")
_exact_keys(signature, {"key_id", "algorithm", "value"}, f"signatures[{index}]")
key_id = _token(signature.get("key_id"), f"signatures[{index}].key_id")
if key_id in seen or signature.get("algorithm") != "ed25519":
raise DistributionError("backup evidence signatures are invalid")
seen.add(key_id)
encoded = signature.get("value")
if not isinstance(encoded, str) or len(encoded) > 256:
raise DistributionError("backup evidence signature value is invalid")
def _reference(raw: object, label: str) -> str:
value = _bounded_text(raw, label, maximum=2048)
if REFERENCE.fullmatch(value) is None or "BEGIN " in value.upper():
raise DistributionError(f"{label} must be an opaque provider reference")
parsed = urlsplit(value)
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise DistributionError(f"{label} must not contain credentials or query data")
return value
def _object(raw: object, label: str) -> dict[str, Any]:
if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw):
raise DistributionError(f"{label} must be an object")
return raw
def _exact_keys(value: Mapping[str, Any], keys: set[str], label: str) -> None:
if set(value) != keys:
missing = sorted(keys - set(value))
extra = sorted(set(value) - keys)
detail = []
if missing:
detail.append("missing " + ", ".join(missing))
if extra:
detail.append("unknown " + ", ".join(extra))
raise DistributionError(f"{label} has invalid fields: {'; '.join(detail)}")
def _timestamp(raw: object, label: str) -> datetime:
value = _bounded_text(raw, label, maximum=64)
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise DistributionError(f"{label} must be an RFC3339 timestamp") from exc
if parsed.tzinfo is None:
raise DistributionError(f"{label} must include a timezone")
return parsed.astimezone(UTC)
def _token(raw: object, label: str) -> str:
value = _bounded_text(raw, label, maximum=128)
if TOKEN.fullmatch(value) is None:
raise DistributionError(f"{label} is invalid")
return value
def _sha256(raw: object, label: str) -> str:
value = _bounded_text(raw, label, maximum=64)
if SHA256.fullmatch(value) is None:
raise DistributionError(f"{label} must be a lowercase SHA-256 digest")
return value
def _image(raw: object, label: str) -> str:
value = _bounded_text(raw, label, maximum=300)
if IMAGE.fullmatch(value) is None:
raise DistributionError(f"{label} must be an OCI image pinned by sha256")
return value
def _bounded_text(raw: object, label: str, *, maximum: int) -> str:
if not isinstance(raw, str) or not raw or len(raw) > maximum or "\n" in raw:
raise DistributionError(f"{label} is invalid")
return raw
def _bounded_integer(
raw: object,
label: str,
*,
maximum: int = 2**63 - 1,
) -> int:
if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0 or raw > maximum:
raise DistributionError(f"{label} is out of bounds")
return raw
__all__ = [
"DEFAULT_MAX_BACKUP_AGE_SECONDS",
"MAX_BACKUP_EVIDENCE_BYTES",
"MAX_BACKUP_KEYRING_BYTES",
"load_backup_evidence",
"load_backup_keyring",
"verify_backup_evidence",
]
@@ -28,7 +28,26 @@ PLAN_FILENAME = "plan.json"
RECEIPT_FILENAME = "receipt.json"
MANIFEST_FILENAME = "distribution-manifest.json"
KEYRING_FILENAME = "distribution-keyring.json"
BACKUP_EVIDENCE_FILENAME = "backup-evidence.json"
BACKUP_KEYRING_FILENAME = "backup-keyring.json"
BACKUP_VERIFICATION_FILENAME = "backup-verification.json"
LOCK_FILENAME = ".deployment.lock"
BACKUP_RUNTIME_ENV_KEYS = (
"GOVOPLAN_BACKUP_EVIDENCE_STATE",
"GOVOPLAN_BACKUP_EVIDENCE_ID",
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID",
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID",
"GOVOPLAN_BACKUP_EVIDENCE_SHA256",
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256",
"GOVOPLAN_BACKUP_CAPTURED_AT",
"GOVOPLAN_BACKUP_EXPIRES_AT",
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT",
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT",
"GOVOPLAN_BACKUP_VERIFIED_AT",
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS",
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS",
"GOVOPLAN_BACKUP_COMPONENT_COUNT",
)
RUNTIME_ENV_KEYS = (
"APP_ENV",
"GOVOPLAN_INSTALL_PROFILE",
@@ -78,6 +97,7 @@ RUNTIME_ENV_KEYS = (
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
*BACKUP_RUNTIME_ENV_KEYS,
)
@@ -95,6 +115,9 @@ class BundlePaths:
receipt: Path
manifest: Path
keyring: Path
backup_evidence: Path
backup_keyring: Path
backup_verification: Path
lock: Path
@@ -116,6 +139,9 @@ def bundle_paths(root: Path) -> BundlePaths:
receipt=resolved / RECEIPT_FILENAME,
manifest=resolved / MANIFEST_FILENAME,
keyring=resolved / KEYRING_FILENAME,
backup_evidence=resolved / BACKUP_EVIDENCE_FILENAME,
backup_keyring=resolved / BACKUP_KEYRING_FILENAME,
backup_verification=resolved / BACKUP_VERIFICATION_FILENAME,
lock=resolved / LOCK_FILENAME,
)
+291 -9
View File
@@ -20,7 +20,14 @@ from typing import Iterator, Mapping, Sequence
from urllib.error import URLError
from urllib.request import urlopen
from .backup_evidence import (
DEFAULT_MAX_BACKUP_AGE_SECONDS,
MAX_BACKUP_EVIDENCE_BYTES,
MAX_BACKUP_KEYRING_BYTES,
verify_backup_evidence,
)
from .bundle import (
BACKUP_RUNTIME_ENV_KEYS,
atomic_write,
bundle_paths,
canonical_json,
@@ -72,7 +79,12 @@ from .kubernetes import (
render_kubernetes,
write_secret_creation_hint,
)
from .planning import DeploymentPlan, build_plan
from .planning import (
DeploymentPlan,
build_plan,
release_change_requires_backup,
verify_stored_backup_evidence,
)
from .recovery import (
DeploymentOperationJournal,
list_operations,
@@ -183,6 +195,22 @@ def build_parser() -> argparse.ArgumentParser:
help="Load verified archives into Docker using fixed image-load commands.",
)
verify_backup = subparsers.add_parser(
"verify-backup",
help="Verify and optionally adopt signed coordinated backup evidence.",
)
_directory_argument(verify_backup)
evidence_source = verify_backup.add_mutually_exclusive_group(required=True)
evidence_source.add_argument("--evidence", type=Path)
evidence_source.add_argument("--evidence-url")
verify_backup.add_argument("--evidence-sha256", required=True)
verify_backup.add_argument("--trusted-keyring", type=Path, required=True)
verify_backup.add_argument(
"--allow-private-evidence-host",
action="store_true",
)
verify_backup.add_argument("--adopt", action="store_true")
kubernetes = subparsers.add_parser(
"render-kubernetes",
help="Export the stateless multi-host runtime for Kubernetes.",
@@ -392,6 +420,8 @@ def main(argv: Sequence[str] | None = None) -> int:
return _verify_release(args)
if args.command == "verify-offline-images":
return _verify_offline_images(args)
if args.command == "verify-backup":
return _verify_backup(args)
if args.command == "render-kubernetes":
return _render_kubernetes(args)
if args.command == "verify-kubernetes":
@@ -534,15 +564,17 @@ def _render_or_doctor(args: argparse.Namespace) -> int:
def _apply(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
spec = load_spec(paths.spec)
if args.allow_unverified_images and spec.profile != "evaluation":
raise ValueError(
"--allow-unverified-images is restricted to evaluation installations"
)
ensure_private_directory(paths.root)
with _deployment_lock(paths.lock):
spec = load_spec(paths.spec)
previous_receipt = _read_json_object(paths.receipt)
backup_required = release_change_requires_backup(spec, previous_receipt)
if args.allow_unverified_images and spec.profile != "evaluation":
raise ValueError(
"--allow-unverified-images is restricted to evaluation installations"
)
secrets = reconcile_runtime_environment(spec, read_env(paths.env))
_write_bundle(spec, paths, secrets)
secrets = _write_bundle(spec, paths, secrets)
plan = build_plan(spec, paths, include_host_checks=True)
_write_plan(paths.plan, plan)
effective_errors = [
@@ -577,6 +609,36 @@ def _apply(args: argparse.Namespace) -> int:
paths,
plan=plan.to_dict(),
)
try:
if backup_required:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=previous_receipt,
)
journal.record(
"backup-evidence-verified",
"succeeded",
dict(backup_summary),
)
else:
journal.record(
"backup-evidence-not-required",
"succeeded",
{"release_change": False},
)
except BaseException as exc:
journal.record(
"backup-evidence-rejected",
"blocked",
{
"phase": "preflight",
"exception_type": type(exc).__name__,
"migration_started": False,
},
)
journal.failed(exc)
raise
compose = [
docker,
"compose",
@@ -624,6 +686,29 @@ def _apply(args: argparse.Namespace) -> int:
"succeeded",
{"services": mutable_runtime_services, "timeout_seconds": 120},
)
if backup_required:
try:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=previous_receipt,
)
except BaseException as exc:
journal.record(
"backup-evidence-rejected",
"blocked",
{
"phase": "migration-boundary",
"exception_type": type(exc).__name__,
"migration_started": False,
},
)
raise
journal.record(
"backup-evidence-reverified",
"succeeded",
dict(backup_summary),
)
journal.migration_started()
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
journal.migration_completed()
@@ -898,6 +983,108 @@ def _verify_offline_images(args: argparse.Namespace) -> int:
return 0
def _verify_backup(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
ensure_private_directory(paths.root)
with _deployment_lock(paths.lock):
return _verify_backup_locked(args, paths)
def _verify_backup_locked(args: argparse.Namespace, paths) -> int:
spec = load_spec(paths.spec)
expected_digest = str(args.evidence_sha256 or "").strip().lower()
if len(expected_digest) != 64 or any(
character not in "0123456789abcdef" for character in expected_digest
):
raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest")
if args.evidence_url:
encoded_evidence = fetch_bounded_https(
str(args.evidence_url),
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
allow_private_host=args.allow_private_evidence_host,
)
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
else:
evidence_path = args.evidence.expanduser().resolve()
encoded_evidence = read_bounded_bytes(
evidence_path,
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
)
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
if encoded_evidence != canonical_distribution_json(evidence):
raise DistributionError("backup evidence is not canonical JSON")
if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest:
raise DistributionError("backup evidence SHA-256 does not match")
keyring_path = args.trusted_keyring.expanduser().resolve()
keyring = load_bounded_json(
keyring_path,
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
)
encoded_keyring = canonical_distribution_json(keyring)
receipt = _read_json_object(paths.receipt)
previous_release = receipt.get("release") if receipt else None
release: Mapping[str, object] = (
previous_release
if isinstance(previous_release, Mapping)
else {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"composition_sha256": spec.release.composition_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
}
)
summary = verify_backup_evidence(
evidence,
keyring,
installation_id=spec.installation_id,
profile=spec.profile,
release=release,
max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS,
)
print(
"Verified coordinated recovery point "
f"{summary['recovery_point_id']} with restore drill "
f"{summary['restore_drill_id']} and trusted key "
f"{summary['signature_key_id']}."
)
if not args.adopt:
return 0
verification = {
"schema_version": 1,
"evidence_sha256": expected_digest,
"keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(),
"signature_key_id": summary["signature_key_id"],
"verified_at": _now(),
"evidence_id": summary["evidence_id"],
"recovery_point_id": summary["recovery_point_id"],
"restore_drill_id": summary["restore_drill_id"],
"release_manifest_sha256": release.get("manifest_sha256"),
"captured_at": summary["captured_at"],
"expires_at": summary["expires_at"],
"restore_started_at": summary["restore_started_at"],
"restore_completed_at": summary["restore_completed_at"],
"measured_rpo_seconds": summary["measured_rpo_seconds"],
"measured_rto_seconds": summary["measured_rto_seconds"],
"component_count": summary["component_count"],
}
atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600)
atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600)
atomic_write(
paths.backup_verification,
canonical_json(verification),
mode=0o600,
)
_write_bundle(
spec,
paths,
reconcile_runtime_environment(spec, read_env(paths.env)),
)
print(f"Adopted signed backup evidence in {paths.root}.")
return 0
def _selected_dependency_images(
spec: InstallationSpec,
*,
@@ -929,6 +1116,30 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
spec = load_spec(paths.spec)
environment = reconcile_runtime_environment(spec, read_env(paths.env))
environment.update(_backup_runtime_environment(spec, paths))
receipt = _read_json_object(paths.receipt)
backup_required = release_change_requires_backup(spec, receipt)
backup_summary: Mapping[str, object] | None = None
evidence_files = (
paths.backup_evidence,
paths.backup_keyring,
paths.backup_verification,
)
if backup_required:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=receipt,
)
elif all(path.is_file() for path in evidence_files):
try:
backup_summary = verify_stored_backup_evidence(
spec,
paths,
receipt=receipt,
)
except (DistributionError, OSError):
backup_summary = None
manifest = render_kubernetes(
spec,
environment,
@@ -936,6 +1147,8 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
secret_name=args.secret_name,
tls_secret_name=args.tls_secret_name,
ingress_class_name=args.ingress_class_name,
backup_required=backup_required,
backup_evidence=backup_summary,
)
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
atomic_write(output, canonical_json(manifest), mode=0o600)
@@ -1027,6 +1240,7 @@ def _deployment_receipt(
return {
"schema_version": 1,
"installation_id": spec.installation_id,
"profile": spec.profile,
"applied_at": _now(),
"spec_sha256": digest_json(spec.to_dict()),
"compose_sha256": digest_json(render_compose(spec)),
@@ -1064,6 +1278,16 @@ def _deployment_receipt(
}
def _read_json_object(path: Path) -> dict[str, object]:
if not path.is_file():
return {}
try:
value = load_bounded_json(path, maximum_bytes=64 * 1024)
except (DistributionError, OSError):
return {}
return value
def _updated_spec(
current: InstallationSpec, args: argparse.Namespace
) -> InstallationSpec:
@@ -1222,10 +1446,12 @@ def _write_bundle(
spec: InstallationSpec,
paths,
secrets: Mapping[str, str],
) -> None:
) -> dict[str, str]:
ensure_private_directory(paths.root)
runtime_environment = dict(secrets)
runtime_environment.update(_backup_runtime_environment(spec, paths))
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
write_env(paths.env, secrets)
write_env(paths.env, runtime_environment)
atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
atomic_write(
paths.load_balancer_config,
@@ -1247,6 +1473,62 @@ def _write_bundle(
render_garage_config().encode("utf-8"),
mode=0o644,
)
return dict(sorted(runtime_environment.items()))
def _backup_runtime_environment(
spec: InstallationSpec,
paths,
) -> dict[str, str]:
values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS}
evidence_files = (
paths.backup_evidence,
paths.backup_keyring,
paths.backup_verification,
)
if not any(path.is_file() for path in evidence_files):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent"
return values
if not all(path.is_file() for path in evidence_files):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
return values
verification = _read_json_object(paths.backup_verification)
try:
summary = verify_stored_backup_evidence(
spec,
paths,
receipt=_read_json_object(paths.receipt),
)
except (DistributionError, OSError):
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
return values
values.update(
{
"GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
"GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]),
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]),
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]),
"GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]),
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str(
verification["release_manifest_sha256"]
),
"GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]),
"GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]),
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]),
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str(
summary["restore_completed_at"]
),
"GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]),
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str(
summary["measured_rpo_seconds"]
),
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str(
summary["measured_rto_seconds"]
),
"GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]),
}
)
return values
def _write_plan(path: Path, plan: DeploymentPlan) -> None:
@@ -74,7 +74,9 @@ def read_bounded_bytes(path: Path, *, maximum_bytes: int) -> bytes:
try:
opened = os.fstat(descriptor)
if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
raise DistributionError(f"trusted JSON file is invalid or too large: {path}")
raise DistributionError(
f"trusted JSON file is invalid or too large: {path}"
)
chunks: list[bytes] = []
total = 0
while True:
@@ -109,7 +111,9 @@ def fetch_bounded_https(
if parsed.scheme != "https" or not parsed.hostname:
raise DistributionError("distribution downloads require an absolute HTTPS URL")
if parsed.username or parsed.password or parsed.fragment:
raise DistributionError("distribution URL must not contain credentials or a fragment")
raise DistributionError(
"distribution URL must not contain credentials or a fragment"
)
if not allow_private_host:
_require_public_host(parsed.hostname)
request = Request(url, headers={"Accept": "application/json"})
@@ -172,9 +176,11 @@ def validate_manifest(
raise DistributionError(
f"distribution channel is {channel!r}, expected {expected_channel!r}"
)
if isinstance(payload.get("sequence"), bool) or not isinstance(
payload.get("sequence"), int
) or int(payload["sequence"]) < 1:
if (
isinstance(payload.get("sequence"), bool)
or not isinstance(payload.get("sequence"), int)
or int(payload["sequence"]) < 1
):
raise DistributionError("distribution sequence must be a positive integer")
_token(payload.get("version"), "version", maximum=128, pattern=TOKEN)
issued = _datetime(payload.get("issued_at"), "issued_at")
@@ -283,11 +289,41 @@ def verify_manifest(
) -> str:
current = (now or datetime.now(UTC)).astimezone(UTC)
validate_manifest(payload, expected_channel=expected_channel, now=current)
keys = _trusted_keys(keyring, now=current)
return verify_signed_document(
payload,
keyring,
purpose="govoplan-runtime-distribution",
label="distribution",
now=current,
openssl=openssl,
)
def verify_signed_document(
payload: Mapping[str, Any],
keyring: Mapping[str, Any],
*,
purpose: str,
label: str,
now: datetime,
openssl: str = "openssl",
) -> str:
keys = _trusted_keys(keyring, now=now, purpose=purpose, label=label)
signed = canonical_signed_payload(payload)
failures: list[str] = []
for item in payload["signatures"]:
signatures = payload.get("signatures")
if not isinstance(signatures, list) or not signatures:
raise DistributionError(f"{label} has no signatures")
for index, raw in enumerate(signatures):
item = _object(raw, f"{label}.signatures[{index}]")
_exact_keys(
item,
required={"key_id", "algorithm", "value"},
label=f"{label}.signatures[{index}]",
)
key_id = str(item["key_id"])
if KEY_ID.fullmatch(key_id) is None or item.get("algorithm") != "ed25519":
raise DistributionError(f"{label} signature is invalid")
public_key = keys.get(key_id)
if public_key is None:
continue
@@ -303,8 +339,10 @@ def verify_manifest(
failures.append(f"{key_id}: {exc}")
continue
return key_id
detail = "; ".join(failures) if failures else "no signature used an active trusted key"
raise DistributionError(f"distribution signature verification failed: {detail}")
detail = (
"; ".join(failures) if failures else "no signature used an active trusted key"
)
raise DistributionError(f"{label} signature verification failed: {detail}")
def verify_manifest_binding(
@@ -319,7 +357,9 @@ def verify_manifest_binding(
dependencies: Mapping[str, str],
) -> None:
if payload.get("channel") != channel or payload.get("version") != version:
raise DistributionError("stored manifest does not match release channel/version")
raise DistributionError(
"stored manifest does not match release channel/version"
)
images = _object(payload.get("images"), "images")
if _object(images.get("api"), "images.api").get("index") != api_image:
raise DistributionError("stored manifest does not match API image")
@@ -372,9 +412,9 @@ def verify_offline_image_index(
archive = root / archive_relative
if reference in references:
raise DistributionError("offline image index contains duplicate references")
if _sha256_regular_file(archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES) != _sha256(
value.get("sha256"), "offline image sha256"
):
if _sha256_regular_file(
archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES
) != _sha256(value.get("sha256"), "offline image sha256"):
raise DistributionError(f"offline image archive digest mismatch: {archive}")
references[reference] = archive
missing = sorted(set(expected_references) - set(references))
@@ -418,19 +458,21 @@ def _trusted_keys(
keyring: Mapping[str, Any],
*,
now: datetime,
purpose: str,
label: str,
) -> dict[str, str]:
_exact_keys(
keyring,
required={"schema_version", "purpose", "keys"},
label="distribution keyring",
label=f"{label} keyring",
)
if keyring.get("schema_version") != "1":
raise DistributionError("unsupported distribution keyring schema_version")
if keyring.get("purpose") != "govoplan-runtime-distribution":
raise DistributionError("distribution keyring has the wrong purpose")
raise DistributionError(f"unsupported {label} keyring schema_version")
if keyring.get("purpose") != purpose:
raise DistributionError(f"{label} keyring has the wrong purpose")
values = keyring.get("keys")
if not isinstance(values, list) or not values:
raise DistributionError("distribution keyring contains no keys")
raise DistributionError(f"{label} keyring contains no keys")
trusted: dict[str, str] = {}
for index, item in enumerate(values):
key = _object(item, f"keyring.keys[{index}]")
@@ -453,11 +495,11 @@ def _trusted_keys(
pattern=KEY_ID,
)
if key_id in trusted:
raise DistributionError("distribution keyring contains duplicate key ids")
raise DistributionError(f"{label} keyring contains duplicate key ids")
if key.get("algorithm") != "ed25519":
raise DistributionError("distribution key must use ed25519")
raise DistributionError(f"{label} key must use ed25519")
if key.get("status") not in {"active", "retired", "revoked"}:
raise DistributionError("distribution key has an invalid status")
raise DistributionError(f"{label} key has an invalid status")
not_before = _datetime(key.get("not_before"), "key.not_before")
expires = _datetime(key.get("expires_at"), "key.expires_at")
public_key = key.get("public_key_pem")
@@ -466,11 +508,11 @@ def _trusted_keys(
or len(public_key.encode("utf-8")) > 8192
or "BEGIN PUBLIC KEY" not in public_key
):
raise DistributionError("distribution key has an invalid public key")
raise DistributionError(f"{label} key has an invalid public key")
if key.get("status") == "active" and not_before <= now < expires:
trusted[key_id] = public_key
if not trusted:
raise DistributionError("distribution keyring has no currently active keys")
raise DistributionError(f"{label} keyring has no currently active keys")
return trusted
@@ -534,13 +576,17 @@ def _require_public_host(hostname: str) -> None:
for value in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM)
}
except OSError as exc:
raise DistributionError(f"distribution host cannot be resolved: {hostname}") from exc
raise DistributionError(
f"distribution host cannot be resolved: {hostname}"
) from exc
if not addresses:
raise DistributionError("distribution host resolved to no addresses")
for value in addresses:
address = ipaddress.ip_address(value)
if not address.is_global:
raise DistributionError("distribution host resolves to a non-public address")
raise DistributionError(
"distribution host resolves to a non-public address"
)
def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
@@ -553,7 +599,9 @@ def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
try:
opened = os.fstat(descriptor)
if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
raise DistributionError(f"immutable artifact is invalid or too large: {path}")
raise DistributionError(
f"immutable artifact is invalid or too large: {path}"
)
while True:
chunk = os.read(descriptor, 1024 * 1024)
if not chunk:
@@ -602,7 +650,11 @@ def _token(
maximum: int,
pattern: re.Pattern[str],
) -> str:
if not isinstance(value, str) or len(value) > maximum or pattern.fullmatch(value) is None:
if (
not isinstance(value, str)
or len(value) > maximum
or pattern.fullmatch(value) is None
):
raise DistributionError(f"{label} is invalid")
return value
@@ -626,7 +678,11 @@ def _sha256(value: object, label: str) -> str:
def _digest_image(value: object, label: str) -> str:
if not isinstance(value, str) or len(value) > 300 or DIGEST_IMAGE.fullmatch(value) is None:
if (
not isinstance(value, str)
or len(value) > 300
or DIGEST_IMAGE.fullmatch(value) is None
):
raise DistributionError(f"{label} must be an OCI image pinned by sha256")
return value
@@ -635,7 +691,12 @@ def _https_url(value: object, label: str) -> str:
if not isinstance(value, str) or len(value) > 2048:
raise DistributionError(f"{label} must be an HTTPS URL")
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
if (
parsed.scheme != "https"
or not parsed.netloc
or parsed.username
or parsed.password
):
raise DistributionError(f"{label} must be an HTTPS URL without credentials")
return value
+26 -1
View File
@@ -10,6 +10,7 @@ import re
from typing import Any, Mapping
from urllib.parse import urlsplit
from .bundle import BACKUP_RUNTIME_ENV_KEYS
from .model import InstallationSpec, image_is_digest_pinned
@@ -55,6 +56,7 @@ _CONFIG_KEYS = (
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
*BACKUP_RUNTIME_ENV_KEYS,
)
_QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
@@ -75,6 +77,8 @@ def render_kubernetes(
secret_name: str = "govoplan-runtime",
tls_secret_name: str = "govoplan-tls",
ingress_class_name: str | None = None,
backup_required: bool = True,
backup_evidence: Mapping[str, object] | None = None,
) -> dict[str, Any]:
"""Render runtime roles only; shared state services stay externally managed."""
@@ -199,6 +203,8 @@ def render_kubernetes(
environment,
"MIGRATION",
),
backup_required=backup_required,
backup_evidence=backup_evidence,
),
]
for pool in worker_pools:
@@ -765,9 +771,28 @@ def _migration_job(
secret_name: str,
service_account: str,
database_environment: Mapping[str, str],
backup_required: bool,
backup_evidence: Mapping[str, object] | None,
) -> dict[str, Any]:
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
job_name = _name_with_suffix(name, f"migrate-{release_key}")
backup_annotations = {
"govoplan.add-ideas.de/backup-required": str(backup_required).lower(),
}
if backup_evidence is not None:
backup_annotations.update(
{
"govoplan.add-ideas.de/backup-evidence-sha256": str(
backup_evidence["evidence_sha256"]
),
"govoplan.add-ideas.de/recovery-point": str(
backup_evidence["recovery_point_id"]
),
"govoplan.add-ideas.de/restore-drill": str(
backup_evidence["restore_drill_id"]
),
}
)
return {
"apiVersion": "batch/v1",
"kind": "Job",
@@ -777,7 +802,7 @@ def _migration_job(
"labels": job_labels,
"annotations": {
"govoplan.add-ideas.de/recovery-mode": "forward-recovery",
"govoplan.add-ideas.de/backup-required": "true",
**backup_annotations,
"argocd.argoproj.io/sync-wave": "-1",
},
},
@@ -3,6 +3,7 @@
from __future__ import annotations
from dataclasses import asdict, dataclass
import hashlib
import json
import os
from pathlib import Path
@@ -18,6 +19,11 @@ from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen
from .backup_evidence import (
MAX_BACKUP_EVIDENCE_BYTES,
MAX_BACKUP_KEYRING_BYTES,
verify_backup_evidence,
)
from .bundle import (
BundlePaths,
canonical_json,
@@ -33,8 +39,11 @@ from .distribution import (
MAX_KEYRING_BYTES,
MAX_MANIFEST_BYTES,
DistributionError,
canonical_json as canonical_distribution_json,
decode_json_bytes,
file_sha256,
load_bounded_json,
read_bounded_bytes,
verify_manifest,
verify_manifest_binding,
)
@@ -232,6 +241,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
)
checks.extend(_distribution_checks(spec, paths))
checks.extend(
_backup_evidence_checks(spec, paths, receipt=_read_receipt(paths.receipt))
)
values = read_env(paths.env)
required = {"MASTER_KEY_B64", "DATABASE_URL"}
@@ -350,6 +362,189 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
return tuple(checks)
def release_change_requires_backup(
spec: InstallationSpec,
receipt: Mapping[str, object],
) -> bool:
if spec.profile != "self-hosted" or not receipt:
return False
previous = receipt.get("release")
if not isinstance(previous, Mapping):
return True
desired = {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"composition_sha256": spec.release.composition_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
}
return any(previous.get(key) != value for key, value in desired.items())
def verify_stored_backup_evidence(
spec: InstallationSpec,
paths: BundlePaths,
*,
receipt: Mapping[str, object],
) -> dict[str, object]:
verification = load_bounded_json(
paths.backup_verification,
maximum_bytes=64 * 1024,
)
expected_fields = {
"schema_version",
"evidence_sha256",
"keyring_sha256",
"signature_key_id",
"verified_at",
"evidence_id",
"recovery_point_id",
"restore_drill_id",
"release_manifest_sha256",
"captured_at",
"expires_at",
"restore_started_at",
"restore_completed_at",
"measured_rpo_seconds",
"measured_rto_seconds",
"component_count",
}
if set(verification) != expected_fields or verification.get("schema_version") != 1:
raise DistributionError("backup verification receipt is malformed")
encoded_evidence = read_bounded_bytes(
paths.backup_evidence,
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
)
encoded_keyring = read_bounded_bytes(
paths.backup_keyring,
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
)
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
keyring = decode_json_bytes(encoded_keyring, label="backup keyring")
if encoded_evidence != canonical_distribution_json(evidence):
raise DistributionError("stored backup evidence is not canonical JSON")
if encoded_keyring != canonical_distribution_json(keyring):
raise DistributionError("stored backup keyring is not canonical JSON")
evidence_digest = hashlib.sha256(encoded_evidence).hexdigest()
keyring_digest = hashlib.sha256(encoded_keyring).hexdigest()
if evidence_digest != verification.get("evidence_sha256"):
raise DistributionError("stored backup evidence digest has changed")
if keyring_digest != verification.get("keyring_sha256"):
raise DistributionError("stored backup keyring digest has changed")
previous_release = receipt.get("release") if receipt else None
expected_release: Mapping[str, object] = (
previous_release
if isinstance(previous_release, Mapping)
else {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"composition_sha256": spec.release.composition_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
}
)
summary = verify_backup_evidence(
evidence,
keyring,
installation_id=spec.installation_id,
profile=spec.profile,
release=expected_release,
)
expected_summary = {
"signature_key_id": verification.get("signature_key_id"),
"evidence_id": verification.get("evidence_id"),
"recovery_point_id": verification.get("recovery_point_id"),
"restore_drill_id": verification.get("restore_drill_id"),
"captured_at": verification.get("captured_at"),
"expires_at": verification.get("expires_at"),
"restore_started_at": verification.get("restore_started_at"),
"restore_completed_at": verification.get("restore_completed_at"),
"measured_rpo_seconds": verification.get("measured_rpo_seconds"),
"measured_rto_seconds": verification.get("measured_rto_seconds"),
"component_count": verification.get("component_count"),
}
for field, expected in expected_summary.items():
if summary.get(field) != expected:
raise DistributionError(
f"backup verification receipt does not match {field!r}"
)
if expected_release.get("manifest_sha256") != verification.get(
"release_manifest_sha256"
):
raise DistributionError("backup verification receipt has another release")
return {
**summary,
"evidence_sha256": evidence_digest,
"keyring_sha256": keyring_digest,
}
def _backup_evidence_checks(
spec: InstallationSpec,
paths: BundlePaths,
*,
receipt: Mapping[str, object],
) -> tuple[Check, ...]:
required = release_change_requires_backup(spec, receipt)
available = all(
path.is_file()
for path in (
paths.backup_evidence,
paths.backup_keyring,
paths.backup_verification,
)
)
if not available:
return (
Check(
"backup.migration_gate",
"error"
if required
else "warning"
if spec.profile == "self-hosted"
else "ok",
(
"A release-changing migration has no verified coordinated backup evidence."
if required
else "No current coordinated backup evidence is adopted."
),
(
"Run verify-backup --adopt after an isolated restore drill."
if spec.profile == "self-hosted"
else ""
),
),
)
try:
summary = verify_stored_backup_evidence(
spec,
paths,
receipt=receipt,
)
except (DistributionError, OSError) as exc:
return (
Check(
"backup.migration_gate",
"error" if required else "warning",
f"Coordinated backup evidence is invalid: {exc}",
"Adopt fresh signed evidence for the currently applied release.",
),
)
return (
Check(
"backup.migration_gate",
"ok",
(
"Release migration is backed by recovery point "
f"{summary['recovery_point_id']} and restore drill "
f"{summary['restore_drill_id']}."
),
),
)
def _ingress_configuration_checks(
spec: InstallationSpec,
paths: BundlePaths,
@@ -29,6 +29,9 @@ _BUNDLE_FILES = (
"existing-proxy.json",
"distribution-manifest.json",
"distribution-keyring.json",
"backup-evidence.json",
"backup-keyring.json",
"backup-verification.json",
"receipt.json",
)
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""Sign and validate provider-produced GovOPlaN backup evidence."""
from __future__ import annotations
import argparse
import base64
import hashlib
from pathlib import Path
import re
import stat
from typing import Any
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from govoplan_deploy.backup_evidence import (
MAX_BACKUP_EVIDENCE_BYTES,
load_backup_keyring,
verify_backup_evidence,
)
from govoplan_deploy.bundle import atomic_write
from govoplan_deploy.distribution import (
canonical_json,
canonical_signed_payload,
load_bounded_json,
)
KEY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
def main() -> int:
parser = argparse.ArgumentParser(
description=(
"Sign a provider-produced backup/restore evidence document and "
"validate it against an independently managed public keyring."
)
)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--trusted-keyring", type=Path, required=True)
parser.add_argument(
"--signing-key",
action="append",
required=True,
metavar="KEY_ID=PRIVATE_PEM",
help="Ed25519 signer; may be repeated during key rotation.",
)
parser.add_argument(
"--replace-signatures",
action="store_true",
help="Replace existing signatures instead of rejecting the input.",
)
args = parser.parse_args()
source = args.input.expanduser().resolve()
payload = load_bounded_json(source, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES)
existing = payload.get("signatures")
if existing not in (None, []) and not args.replace_signatures:
raise SystemExit("input already contains signatures; use --replace-signatures")
signers = [_load_signer(value) for value in args.signing_key]
if len({key_id for key_id, _ in signers}) != len(signers):
raise SystemExit("duplicate signing key id")
payload["signatures"] = []
signed = canonical_signed_payload(payload)
payload["signatures"] = [
{
"key_id": key_id,
"algorithm": "ed25519",
"value": base64.b64encode(private_key.sign(signed)).decode("ascii"),
}
for key_id, private_key in signers
]
keyring = load_backup_keyring(args.trusted_keyring.expanduser().resolve())
release = payload.get("release")
if not isinstance(release, dict):
raise SystemExit("input release must be an object")
verify_backup_evidence(
payload,
keyring,
installation_id=str(payload.get("installation_id") or ""),
profile=str(
_object(payload.get("deployment_subject"), "deployment_subject").get(
"profile"
)
or ""
),
release=release,
)
encoded = canonical_json(payload)
output = args.output.expanduser().resolve()
atomic_write(output, encoded, mode=0o600)
print(f"Wrote {output}")
print(f"SHA256 {hashlib.sha256(encoded).hexdigest()}")
return 0
def _load_signer(value: str) -> tuple[str, Ed25519PrivateKey]:
key_id, separator, raw_path = value.partition("=")
if not separator or KEY_ID.fullmatch(key_id) is None or not raw_path:
raise SystemExit("--signing-key must use KEY_ID=/path/to/private.pem")
path = Path(raw_path).expanduser().resolve()
mode = stat.S_IMODE(path.stat().st_mode)
if mode & 0o077:
raise SystemExit(
f"private signing key must not be group/world accessible: {path}"
)
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
if not isinstance(private_key, Ed25519PrivateKey):
raise SystemExit(f"signing key is not Ed25519: {path}")
return key_id, private_key
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise SystemExit(f"input {label} must be an object")
return value
if __name__ == "__main__":
raise SystemExit(main())