Enforce signed backup evidence before migrations
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -18,6 +19,11 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .backup_evidence import (
|
||||
MAX_BACKUP_EVIDENCE_BYTES,
|
||||
MAX_BACKUP_KEYRING_BYTES,
|
||||
verify_backup_evidence,
|
||||
)
|
||||
from .bundle import (
|
||||
BundlePaths,
|
||||
canonical_json,
|
||||
@@ -33,8 +39,11 @@ from .distribution import (
|
||||
MAX_KEYRING_BYTES,
|
||||
MAX_MANIFEST_BYTES,
|
||||
DistributionError,
|
||||
canonical_json as canonical_distribution_json,
|
||||
decode_json_bytes,
|
||||
file_sha256,
|
||||
load_bounded_json,
|
||||
read_bounded_bytes,
|
||||
verify_manifest,
|
||||
verify_manifest_binding,
|
||||
)
|
||||
@@ -232,6 +241,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
|
||||
)
|
||||
|
||||
checks.extend(_distribution_checks(spec, paths))
|
||||
checks.extend(
|
||||
_backup_evidence_checks(spec, paths, receipt=_read_receipt(paths.receipt))
|
||||
)
|
||||
|
||||
values = read_env(paths.env)
|
||||
required = {"MASTER_KEY_B64", "DATABASE_URL"}
|
||||
@@ -350,6 +362,189 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
|
||||
return tuple(checks)
|
||||
|
||||
|
||||
def release_change_requires_backup(
|
||||
spec: InstallationSpec,
|
||||
receipt: Mapping[str, object],
|
||||
) -> bool:
|
||||
if spec.profile != "self-hosted" or not receipt:
|
||||
return False
|
||||
previous = receipt.get("release")
|
||||
if not isinstance(previous, Mapping):
|
||||
return True
|
||||
desired = {
|
||||
"channel": spec.release.channel,
|
||||
"version": spec.release.version,
|
||||
"manifest_sha256": spec.release.manifest_sha256,
|
||||
"composition_sha256": spec.release.composition_sha256,
|
||||
"api_image": spec.release.api_image,
|
||||
"web_image": spec.release.web_image,
|
||||
}
|
||||
return any(previous.get(key) != value for key, value in desired.items())
|
||||
|
||||
|
||||
def verify_stored_backup_evidence(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
receipt: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
verification = load_bounded_json(
|
||||
paths.backup_verification,
|
||||
maximum_bytes=64 * 1024,
|
||||
)
|
||||
expected_fields = {
|
||||
"schema_version",
|
||||
"evidence_sha256",
|
||||
"keyring_sha256",
|
||||
"signature_key_id",
|
||||
"verified_at",
|
||||
"evidence_id",
|
||||
"recovery_point_id",
|
||||
"restore_drill_id",
|
||||
"release_manifest_sha256",
|
||||
"captured_at",
|
||||
"expires_at",
|
||||
"restore_started_at",
|
||||
"restore_completed_at",
|
||||
"measured_rpo_seconds",
|
||||
"measured_rto_seconds",
|
||||
"component_count",
|
||||
}
|
||||
if set(verification) != expected_fields or verification.get("schema_version") != 1:
|
||||
raise DistributionError("backup verification receipt is malformed")
|
||||
encoded_evidence = read_bounded_bytes(
|
||||
paths.backup_evidence,
|
||||
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
||||
)
|
||||
encoded_keyring = read_bounded_bytes(
|
||||
paths.backup_keyring,
|
||||
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
|
||||
)
|
||||
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
||||
keyring = decode_json_bytes(encoded_keyring, label="backup keyring")
|
||||
if encoded_evidence != canonical_distribution_json(evidence):
|
||||
raise DistributionError("stored backup evidence is not canonical JSON")
|
||||
if encoded_keyring != canonical_distribution_json(keyring):
|
||||
raise DistributionError("stored backup keyring is not canonical JSON")
|
||||
evidence_digest = hashlib.sha256(encoded_evidence).hexdigest()
|
||||
keyring_digest = hashlib.sha256(encoded_keyring).hexdigest()
|
||||
if evidence_digest != verification.get("evidence_sha256"):
|
||||
raise DistributionError("stored backup evidence digest has changed")
|
||||
if keyring_digest != verification.get("keyring_sha256"):
|
||||
raise DistributionError("stored backup keyring digest has changed")
|
||||
previous_release = receipt.get("release") if receipt else None
|
||||
expected_release: Mapping[str, object] = (
|
||||
previous_release
|
||||
if isinstance(previous_release, Mapping)
|
||||
else {
|
||||
"channel": spec.release.channel,
|
||||
"version": spec.release.version,
|
||||
"manifest_sha256": spec.release.manifest_sha256,
|
||||
"composition_sha256": spec.release.composition_sha256,
|
||||
"api_image": spec.release.api_image,
|
||||
"web_image": spec.release.web_image,
|
||||
}
|
||||
)
|
||||
summary = verify_backup_evidence(
|
||||
evidence,
|
||||
keyring,
|
||||
installation_id=spec.installation_id,
|
||||
profile=spec.profile,
|
||||
release=expected_release,
|
||||
)
|
||||
expected_summary = {
|
||||
"signature_key_id": verification.get("signature_key_id"),
|
||||
"evidence_id": verification.get("evidence_id"),
|
||||
"recovery_point_id": verification.get("recovery_point_id"),
|
||||
"restore_drill_id": verification.get("restore_drill_id"),
|
||||
"captured_at": verification.get("captured_at"),
|
||||
"expires_at": verification.get("expires_at"),
|
||||
"restore_started_at": verification.get("restore_started_at"),
|
||||
"restore_completed_at": verification.get("restore_completed_at"),
|
||||
"measured_rpo_seconds": verification.get("measured_rpo_seconds"),
|
||||
"measured_rto_seconds": verification.get("measured_rto_seconds"),
|
||||
"component_count": verification.get("component_count"),
|
||||
}
|
||||
for field, expected in expected_summary.items():
|
||||
if summary.get(field) != expected:
|
||||
raise DistributionError(
|
||||
f"backup verification receipt does not match {field!r}"
|
||||
)
|
||||
if expected_release.get("manifest_sha256") != verification.get(
|
||||
"release_manifest_sha256"
|
||||
):
|
||||
raise DistributionError("backup verification receipt has another release")
|
||||
return {
|
||||
**summary,
|
||||
"evidence_sha256": evidence_digest,
|
||||
"keyring_sha256": keyring_digest,
|
||||
}
|
||||
|
||||
|
||||
def _backup_evidence_checks(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
receipt: Mapping[str, object],
|
||||
) -> tuple[Check, ...]:
|
||||
required = release_change_requires_backup(spec, receipt)
|
||||
available = all(
|
||||
path.is_file()
|
||||
for path in (
|
||||
paths.backup_evidence,
|
||||
paths.backup_keyring,
|
||||
paths.backup_verification,
|
||||
)
|
||||
)
|
||||
if not available:
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"error"
|
||||
if required
|
||||
else "warning"
|
||||
if spec.profile == "self-hosted"
|
||||
else "ok",
|
||||
(
|
||||
"A release-changing migration has no verified coordinated backup evidence."
|
||||
if required
|
||||
else "No current coordinated backup evidence is adopted."
|
||||
),
|
||||
(
|
||||
"Run verify-backup --adopt after an isolated restore drill."
|
||||
if spec.profile == "self-hosted"
|
||||
else ""
|
||||
),
|
||||
),
|
||||
)
|
||||
try:
|
||||
summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=receipt,
|
||||
)
|
||||
except (DistributionError, OSError) as exc:
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"error" if required else "warning",
|
||||
f"Coordinated backup evidence is invalid: {exc}",
|
||||
"Adopt fresh signed evidence for the currently applied release.",
|
||||
),
|
||||
)
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"ok",
|
||||
(
|
||||
"Release migration is backed by recovery point "
|
||||
f"{summary['recovery_point_id']} and restore drill "
|
||||
f"{summary['restore_drill_id']}."
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _ingress_configuration_checks(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
|
||||
Reference in New Issue
Block a user