Enforce signed backup evidence before migrations
This commit is contained in:
@@ -0,0 +1,500 @@
|
||||
"""Signed, provider-neutral backup and isolated-restore evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any, Mapping
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .distribution import (
|
||||
DistributionError,
|
||||
load_bounded_json,
|
||||
verify_signed_document,
|
||||
)
|
||||
|
||||
|
||||
MAX_BACKUP_EVIDENCE_BYTES = 1024 * 1024
|
||||
MAX_BACKUP_KEYRING_BYTES = 1024 * 1024
|
||||
DEFAULT_MAX_BACKUP_AGE_SECONDS = 24 * 60 * 60
|
||||
MAX_COORDINATION_SKEW_SECONDS = 5 * 60
|
||||
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
TOKEN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
|
||||
REFERENCE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:[^\s]{1,2040}$")
|
||||
|
||||
|
||||
def load_backup_evidence(path: Path) -> dict[str, Any]:
|
||||
return load_bounded_json(path, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES)
|
||||
|
||||
|
||||
def load_backup_keyring(path: Path) -> dict[str, Any]:
|
||||
return load_bounded_json(path, maximum_bytes=MAX_BACKUP_KEYRING_BYTES)
|
||||
|
||||
|
||||
def verify_backup_evidence(
|
||||
payload: Mapping[str, Any],
|
||||
keyring: Mapping[str, Any],
|
||||
*,
|
||||
installation_id: str,
|
||||
profile: str,
|
||||
release: Mapping[str, object],
|
||||
now: datetime | None = None,
|
||||
max_age_seconds: int = DEFAULT_MAX_BACKUP_AGE_SECONDS,
|
||||
openssl: str = "openssl",
|
||||
) -> dict[str, object]:
|
||||
current = (now or datetime.now(UTC)).astimezone(UTC)
|
||||
values = _validate_payload(payload, now=current, max_age_seconds=max_age_seconds)
|
||||
if payload.get("installation_id") != installation_id:
|
||||
raise DistributionError("backup evidence belongs to another installation")
|
||||
subject = _object(payload.get("deployment_subject"), "deployment_subject")
|
||||
if subject.get("profile") != profile:
|
||||
raise DistributionError("backup evidence belongs to another deployment profile")
|
||||
evidence_release = _object(payload.get("release"), "release")
|
||||
for field in (
|
||||
"channel",
|
||||
"version",
|
||||
"manifest_sha256",
|
||||
"composition_sha256",
|
||||
"api_image",
|
||||
"web_image",
|
||||
):
|
||||
if evidence_release.get(field) != release.get(field):
|
||||
raise DistributionError(
|
||||
f"backup evidence does not match release field {field!r}"
|
||||
)
|
||||
key_id = verify_signed_document(
|
||||
payload,
|
||||
keyring,
|
||||
purpose="govoplan-backup-evidence",
|
||||
label="backup evidence",
|
||||
now=current,
|
||||
openssl=openssl,
|
||||
)
|
||||
recovery_point = _object(payload.get("recovery_point"), "recovery_point")
|
||||
restore = _object(payload.get("restore_drill"), "restore_drill")
|
||||
return {
|
||||
"evidence_id": payload["evidence_id"],
|
||||
"recovery_point_id": recovery_point["id"],
|
||||
"captured_at": recovery_point["captured_at"],
|
||||
"expires_at": payload["expires_at"],
|
||||
"restore_drill_id": restore["drill_id"],
|
||||
"restore_started_at": restore["started_at"],
|
||||
"restore_completed_at": restore["completed_at"],
|
||||
"measured_rpo_seconds": restore["measured_rpo_seconds"],
|
||||
"measured_rto_seconds": restore["measured_rto_seconds"],
|
||||
"signature_key_id": key_id,
|
||||
"component_count": values["component_count"],
|
||||
}
|
||||
|
||||
|
||||
def _validate_payload(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
now: datetime,
|
||||
max_age_seconds: int,
|
||||
) -> dict[str, int]:
|
||||
if max_age_seconds < 60 or max_age_seconds > 30 * 24 * 60 * 60:
|
||||
raise DistributionError("backup maximum age is out of bounds")
|
||||
_exact_keys(
|
||||
payload,
|
||||
{
|
||||
"schema_version",
|
||||
"evidence_id",
|
||||
"installation_id",
|
||||
"deployment_subject",
|
||||
"release",
|
||||
"recovery_point",
|
||||
"components",
|
||||
"restore_drill",
|
||||
"issued_at",
|
||||
"expires_at",
|
||||
"revoked",
|
||||
"signatures",
|
||||
},
|
||||
"backup evidence",
|
||||
)
|
||||
if payload.get("schema_version") != "1":
|
||||
raise DistributionError("unsupported backup evidence schema_version")
|
||||
_token(payload.get("evidence_id"), "evidence_id")
|
||||
_token(payload.get("installation_id"), "installation_id")
|
||||
issued = _timestamp(payload.get("issued_at"), "issued_at")
|
||||
expires = _timestamp(payload.get("expires_at"), "expires_at")
|
||||
if issued > now or expires <= issued or expires <= now:
|
||||
raise DistributionError("backup evidence is not currently valid")
|
||||
if payload.get("revoked") is not False:
|
||||
raise DistributionError("backup evidence is revoked")
|
||||
|
||||
subject = _object(payload.get("deployment_subject"), "deployment_subject")
|
||||
_exact_keys(subject, {"profile", "topology", "subject_ref"}, "deployment_subject")
|
||||
if subject.get("profile") not in {"evaluation", "self-hosted"}:
|
||||
raise DistributionError("deployment_subject.profile is invalid")
|
||||
_token(subject.get("topology"), "deployment_subject.topology")
|
||||
_reference(subject.get("subject_ref"), "deployment_subject.subject_ref")
|
||||
|
||||
release = _object(payload.get("release"), "release")
|
||||
_exact_keys(
|
||||
release,
|
||||
{
|
||||
"channel",
|
||||
"version",
|
||||
"manifest_sha256",
|
||||
"composition_sha256",
|
||||
"api_image",
|
||||
"web_image",
|
||||
},
|
||||
"release",
|
||||
)
|
||||
_token(release.get("channel"), "release.channel")
|
||||
_token(release.get("version"), "release.version")
|
||||
_sha256(release.get("manifest_sha256"), "release.manifest_sha256")
|
||||
_sha256(release.get("composition_sha256"), "release.composition_sha256")
|
||||
_image(release.get("api_image"), "release.api_image")
|
||||
_image(release.get("web_image"), "release.web_image")
|
||||
|
||||
recovery = _object(payload.get("recovery_point"), "recovery_point")
|
||||
_exact_keys(
|
||||
recovery,
|
||||
{"id", "captured_at", "consistency", "rpo_seconds", "write_fence"},
|
||||
"recovery_point",
|
||||
)
|
||||
recovery_id = _token(recovery.get("id"), "recovery_point.id")
|
||||
captured = _timestamp(recovery.get("captured_at"), "recovery_point.captured_at")
|
||||
age = (now - captured).total_seconds()
|
||||
if age < 0 or age > max_age_seconds:
|
||||
raise DistributionError("backup recovery point is stale or in the future")
|
||||
if recovery.get("consistency") not in {
|
||||
"provider-atomic",
|
||||
"application-quiesced",
|
||||
"transaction-consistent",
|
||||
}:
|
||||
raise DistributionError("recovery_point.consistency is invalid")
|
||||
declared_rpo = _bounded_integer(
|
||||
recovery.get("rpo_seconds"),
|
||||
"recovery_point.rpo_seconds",
|
||||
maximum=30 * 24 * 60 * 60,
|
||||
)
|
||||
fence = _object(recovery.get("write_fence"), "recovery_point.write_fence")
|
||||
_exact_keys(
|
||||
fence,
|
||||
{"mode", "token_sha256", "established_at"},
|
||||
"recovery_point.write_fence",
|
||||
)
|
||||
if fence.get("mode") not in {
|
||||
"provider-snapshot",
|
||||
"application-quiesce",
|
||||
"transaction-boundary",
|
||||
}:
|
||||
raise DistributionError("recovery_point.write_fence.mode is invalid")
|
||||
_sha256(fence.get("token_sha256"), "recovery_point.write_fence.token_sha256")
|
||||
established = _timestamp(
|
||||
fence.get("established_at"),
|
||||
"recovery_point.write_fence.established_at",
|
||||
)
|
||||
if abs((captured - established).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS:
|
||||
raise DistributionError(
|
||||
"backup write fence is not coordinated with recovery point"
|
||||
)
|
||||
|
||||
components = _object(payload.get("components"), "components")
|
||||
_exact_keys(
|
||||
components,
|
||||
{"database", "objects", "configuration", "key_custody"},
|
||||
"components",
|
||||
)
|
||||
captured_components = [
|
||||
_database_component(components.get("database")),
|
||||
_objects_component(components.get("objects")),
|
||||
_configuration_component(components.get("configuration")),
|
||||
_key_custody_component(components.get("key_custody")),
|
||||
]
|
||||
if any(
|
||||
abs((component_time - captured).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS
|
||||
for component_time in captured_components
|
||||
):
|
||||
raise DistributionError("backup components do not share one recovery point")
|
||||
|
||||
restore = _object(payload.get("restore_drill"), "restore_drill")
|
||||
_exact_keys(
|
||||
restore,
|
||||
{
|
||||
"drill_id",
|
||||
"recovery_point_id",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"isolated_target_ref",
|
||||
"release_manifest_sha256",
|
||||
"migration_heads_sha256",
|
||||
"representative_object_manifest_sha256",
|
||||
"database_verified",
|
||||
"objects_verified",
|
||||
"configuration_verified",
|
||||
"key_custody_verified",
|
||||
"semantic_checks",
|
||||
"measured_rpo_seconds",
|
||||
"measured_rto_seconds",
|
||||
"evidence_ref",
|
||||
},
|
||||
"restore_drill",
|
||||
)
|
||||
_token(restore.get("drill_id"), "restore_drill.drill_id")
|
||||
if restore.get("recovery_point_id") != recovery_id:
|
||||
raise DistributionError("restore drill used another recovery point")
|
||||
started = _timestamp(restore.get("started_at"), "restore_drill.started_at")
|
||||
completed = _timestamp(restore.get("completed_at"), "restore_drill.completed_at")
|
||||
if started < captured or completed < started or completed > issued:
|
||||
raise DistributionError(
|
||||
"restore drill completion is outside evidence chronology"
|
||||
)
|
||||
_reference(restore.get("isolated_target_ref"), "restore_drill.isolated_target_ref")
|
||||
_reference(restore.get("evidence_ref"), "restore_drill.evidence_ref")
|
||||
for field in (
|
||||
"release_manifest_sha256",
|
||||
"migration_heads_sha256",
|
||||
"representative_object_manifest_sha256",
|
||||
):
|
||||
_sha256(restore.get(field), f"restore_drill.{field}")
|
||||
if restore.get("release_manifest_sha256") != release.get("manifest_sha256"):
|
||||
raise DistributionError("restore drill used another immutable release")
|
||||
for field in (
|
||||
"database_verified",
|
||||
"objects_verified",
|
||||
"configuration_verified",
|
||||
"key_custody_verified",
|
||||
):
|
||||
if restore.get(field) is not True:
|
||||
raise DistributionError(f"restore_drill.{field} must be true")
|
||||
semantic = restore.get("semantic_checks")
|
||||
if not isinstance(semantic, list) or not semantic or len(semantic) > 128:
|
||||
raise DistributionError("restore_drill.semantic_checks must not be empty")
|
||||
seen_checks: set[str] = set()
|
||||
for index, raw in enumerate(semantic):
|
||||
check = _object(raw, f"restore_drill.semantic_checks[{index}]")
|
||||
_exact_keys(
|
||||
check,
|
||||
{"id", "status", "evidence_ref"},
|
||||
f"restore_drill.semantic_checks[{index}]",
|
||||
)
|
||||
check_id = _token(check.get("id"), f"semantic_checks[{index}].id")
|
||||
if check_id in seen_checks or check.get("status") != "passed":
|
||||
raise DistributionError("restore drill semantic checks are invalid")
|
||||
seen_checks.add(check_id)
|
||||
_reference(check.get("evidence_ref"), f"semantic_checks[{index}].evidence_ref")
|
||||
measured_rpo = _bounded_integer(
|
||||
restore.get("measured_rpo_seconds"),
|
||||
"restore_drill.measured_rpo_seconds",
|
||||
maximum=30 * 24 * 60 * 60,
|
||||
)
|
||||
measured_rto = _bounded_integer(
|
||||
restore.get("measured_rto_seconds"),
|
||||
"restore_drill.measured_rto_seconds",
|
||||
maximum=30 * 24 * 60 * 60,
|
||||
)
|
||||
if abs((completed - started).total_seconds() - measured_rto) > 5:
|
||||
raise DistributionError("restore drill RTO does not match its timestamps")
|
||||
if measured_rpo > declared_rpo:
|
||||
raise DistributionError(
|
||||
"restore drill exceeds the declared recovery point objective"
|
||||
)
|
||||
_validate_signatures(payload.get("signatures"))
|
||||
return {"component_count": len(captured_components)}
|
||||
|
||||
|
||||
def _database_component(raw: object) -> datetime:
|
||||
value = _object(raw, "components.database")
|
||||
_exact_keys(
|
||||
value,
|
||||
{
|
||||
"provider",
|
||||
"artifact_ref",
|
||||
"artifact_sha256",
|
||||
"snapshot_id",
|
||||
"lsn",
|
||||
"protected",
|
||||
"encryption_key_ref",
|
||||
"captured_at",
|
||||
},
|
||||
"components.database",
|
||||
)
|
||||
_common_artifact(value, "components.database")
|
||||
_token(value.get("snapshot_id"), "components.database.snapshot_id")
|
||||
_bounded_text(value.get("lsn"), "components.database.lsn", maximum=256)
|
||||
return _timestamp(value.get("captured_at"), "components.database.captured_at")
|
||||
|
||||
|
||||
def _objects_component(raw: object) -> datetime:
|
||||
value = _object(raw, "components.objects")
|
||||
_exact_keys(
|
||||
value,
|
||||
{
|
||||
"provider",
|
||||
"artifact_ref",
|
||||
"manifest_sha256",
|
||||
"version_id",
|
||||
"object_count",
|
||||
"total_bytes",
|
||||
"protected",
|
||||
"encryption_key_ref",
|
||||
"captured_at",
|
||||
},
|
||||
"components.objects",
|
||||
)
|
||||
_token(value.get("provider"), "components.objects.provider")
|
||||
_reference(value.get("artifact_ref"), "components.objects.artifact_ref")
|
||||
_sha256(value.get("manifest_sha256"), "components.objects.manifest_sha256")
|
||||
_token(value.get("version_id"), "components.objects.version_id")
|
||||
_bounded_integer(value.get("object_count"), "components.objects.object_count")
|
||||
_bounded_integer(value.get("total_bytes"), "components.objects.total_bytes")
|
||||
_protected_key_reference(value, "components.objects")
|
||||
return _timestamp(value.get("captured_at"), "components.objects.captured_at")
|
||||
|
||||
|
||||
def _configuration_component(raw: object) -> datetime:
|
||||
value = _object(raw, "components.configuration")
|
||||
_exact_keys(
|
||||
value,
|
||||
{
|
||||
"artifact_ref",
|
||||
"sha256",
|
||||
"protected",
|
||||
"encryption_key_ref",
|
||||
"captured_at",
|
||||
},
|
||||
"components.configuration",
|
||||
)
|
||||
_reference(value.get("artifact_ref"), "components.configuration.artifact_ref")
|
||||
_sha256(value.get("sha256"), "components.configuration.sha256")
|
||||
_protected_key_reference(value, "components.configuration")
|
||||
return _timestamp(value.get("captured_at"), "components.configuration.captured_at")
|
||||
|
||||
|
||||
def _key_custody_component(raw: object) -> datetime:
|
||||
value = _object(raw, "components.key_custody")
|
||||
_exact_keys(
|
||||
value,
|
||||
{"provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"},
|
||||
"components.key_custody",
|
||||
)
|
||||
_token(value.get("provider"), "components.key_custody.provider")
|
||||
_reference(value.get("keyset_ref"), "components.key_custody.keyset_ref")
|
||||
_token(value.get("keyset_version"), "components.key_custody.keyset_version")
|
||||
if value.get("recoverable") is not True:
|
||||
raise DistributionError("components.key_custody.recoverable must be true")
|
||||
return _timestamp(value.get("captured_at"), "components.key_custody.captured_at")
|
||||
|
||||
|
||||
def _common_artifact(value: Mapping[str, Any], label: str) -> None:
|
||||
_token(value.get("provider"), f"{label}.provider")
|
||||
_reference(value.get("artifact_ref"), f"{label}.artifact_ref")
|
||||
_sha256(value.get("artifact_sha256"), f"{label}.artifact_sha256")
|
||||
_protected_key_reference(value, label)
|
||||
|
||||
|
||||
def _protected_key_reference(value: Mapping[str, Any], label: str) -> None:
|
||||
if value.get("protected") is not True:
|
||||
raise DistributionError(f"{label}.protected must be true")
|
||||
_reference(value.get("encryption_key_ref"), f"{label}.encryption_key_ref")
|
||||
|
||||
|
||||
def _validate_signatures(raw: object) -> None:
|
||||
if not isinstance(raw, list) or not raw or len(raw) > 16:
|
||||
raise DistributionError("backup evidence signatures must not be empty")
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(raw):
|
||||
signature = _object(item, f"signatures[{index}]")
|
||||
_exact_keys(signature, {"key_id", "algorithm", "value"}, f"signatures[{index}]")
|
||||
key_id = _token(signature.get("key_id"), f"signatures[{index}].key_id")
|
||||
if key_id in seen or signature.get("algorithm") != "ed25519":
|
||||
raise DistributionError("backup evidence signatures are invalid")
|
||||
seen.add(key_id)
|
||||
encoded = signature.get("value")
|
||||
if not isinstance(encoded, str) or len(encoded) > 256:
|
||||
raise DistributionError("backup evidence signature value is invalid")
|
||||
|
||||
|
||||
def _reference(raw: object, label: str) -> str:
|
||||
value = _bounded_text(raw, label, maximum=2048)
|
||||
if REFERENCE.fullmatch(value) is None or "BEGIN " in value.upper():
|
||||
raise DistributionError(f"{label} must be an opaque provider reference")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise DistributionError(f"{label} must not contain credentials or query data")
|
||||
return value
|
||||
|
||||
|
||||
def _object(raw: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw):
|
||||
raise DistributionError(f"{label} must be an object")
|
||||
return raw
|
||||
|
||||
|
||||
def _exact_keys(value: Mapping[str, Any], keys: set[str], label: str) -> None:
|
||||
if set(value) != keys:
|
||||
missing = sorted(keys - set(value))
|
||||
extra = sorted(set(value) - keys)
|
||||
detail = []
|
||||
if missing:
|
||||
detail.append("missing " + ", ".join(missing))
|
||||
if extra:
|
||||
detail.append("unknown " + ", ".join(extra))
|
||||
raise DistributionError(f"{label} has invalid fields: {'; '.join(detail)}")
|
||||
|
||||
|
||||
def _timestamp(raw: object, label: str) -> datetime:
|
||||
value = _bounded_text(raw, label, maximum=64)
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise DistributionError(f"{label} must be an RFC3339 timestamp") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise DistributionError(f"{label} must include a timezone")
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _token(raw: object, label: str) -> str:
|
||||
value = _bounded_text(raw, label, maximum=128)
|
||||
if TOKEN.fullmatch(value) is None:
|
||||
raise DistributionError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(raw: object, label: str) -> str:
|
||||
value = _bounded_text(raw, label, maximum=64)
|
||||
if SHA256.fullmatch(value) is None:
|
||||
raise DistributionError(f"{label} must be a lowercase SHA-256 digest")
|
||||
return value
|
||||
|
||||
|
||||
def _image(raw: object, label: str) -> str:
|
||||
value = _bounded_text(raw, label, maximum=300)
|
||||
if IMAGE.fullmatch(value) is None:
|
||||
raise DistributionError(f"{label} must be an OCI image pinned by sha256")
|
||||
return value
|
||||
|
||||
|
||||
def _bounded_text(raw: object, label: str, *, maximum: int) -> str:
|
||||
if not isinstance(raw, str) or not raw or len(raw) > maximum or "\n" in raw:
|
||||
raise DistributionError(f"{label} is invalid")
|
||||
return raw
|
||||
|
||||
|
||||
def _bounded_integer(
|
||||
raw: object,
|
||||
label: str,
|
||||
*,
|
||||
maximum: int = 2**63 - 1,
|
||||
) -> int:
|
||||
if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0 or raw > maximum:
|
||||
raise DistributionError(f"{label} is out of bounds")
|
||||
return raw
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_MAX_BACKUP_AGE_SECONDS",
|
||||
"MAX_BACKUP_EVIDENCE_BYTES",
|
||||
"MAX_BACKUP_KEYRING_BYTES",
|
||||
"load_backup_evidence",
|
||||
"load_backup_keyring",
|
||||
"verify_backup_evidence",
|
||||
]
|
||||
@@ -28,7 +28,26 @@ PLAN_FILENAME = "plan.json"
|
||||
RECEIPT_FILENAME = "receipt.json"
|
||||
MANIFEST_FILENAME = "distribution-manifest.json"
|
||||
KEYRING_FILENAME = "distribution-keyring.json"
|
||||
BACKUP_EVIDENCE_FILENAME = "backup-evidence.json"
|
||||
BACKUP_KEYRING_FILENAME = "backup-keyring.json"
|
||||
BACKUP_VERIFICATION_FILENAME = "backup-verification.json"
|
||||
LOCK_FILENAME = ".deployment.lock"
|
||||
BACKUP_RUNTIME_ENV_KEYS = (
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_STATE",
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_ID",
|
||||
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID",
|
||||
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID",
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_SHA256",
|
||||
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256",
|
||||
"GOVOPLAN_BACKUP_CAPTURED_AT",
|
||||
"GOVOPLAN_BACKUP_EXPIRES_AT",
|
||||
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT",
|
||||
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT",
|
||||
"GOVOPLAN_BACKUP_VERIFIED_AT",
|
||||
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS",
|
||||
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS",
|
||||
"GOVOPLAN_BACKUP_COMPONENT_COUNT",
|
||||
)
|
||||
RUNTIME_ENV_KEYS = (
|
||||
"APP_ENV",
|
||||
"GOVOPLAN_INSTALL_PROFILE",
|
||||
@@ -78,6 +97,7 @@ RUNTIME_ENV_KEYS = (
|
||||
"FILE_STORAGE_S3_BUCKET",
|
||||
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
||||
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
||||
*BACKUP_RUNTIME_ENV_KEYS,
|
||||
)
|
||||
|
||||
|
||||
@@ -95,6 +115,9 @@ class BundlePaths:
|
||||
receipt: Path
|
||||
manifest: Path
|
||||
keyring: Path
|
||||
backup_evidence: Path
|
||||
backup_keyring: Path
|
||||
backup_verification: Path
|
||||
lock: Path
|
||||
|
||||
|
||||
@@ -116,6 +139,9 @@ def bundle_paths(root: Path) -> BundlePaths:
|
||||
receipt=resolved / RECEIPT_FILENAME,
|
||||
manifest=resolved / MANIFEST_FILENAME,
|
||||
keyring=resolved / KEYRING_FILENAME,
|
||||
backup_evidence=resolved / BACKUP_EVIDENCE_FILENAME,
|
||||
backup_keyring=resolved / BACKUP_KEYRING_FILENAME,
|
||||
backup_verification=resolved / BACKUP_VERIFICATION_FILENAME,
|
||||
lock=resolved / LOCK_FILENAME,
|
||||
)
|
||||
|
||||
|
||||
@@ -20,7 +20,14 @@ from typing import Iterator, Mapping, Sequence
|
||||
from urllib.error import URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
from .backup_evidence import (
|
||||
DEFAULT_MAX_BACKUP_AGE_SECONDS,
|
||||
MAX_BACKUP_EVIDENCE_BYTES,
|
||||
MAX_BACKUP_KEYRING_BYTES,
|
||||
verify_backup_evidence,
|
||||
)
|
||||
from .bundle import (
|
||||
BACKUP_RUNTIME_ENV_KEYS,
|
||||
atomic_write,
|
||||
bundle_paths,
|
||||
canonical_json,
|
||||
@@ -72,7 +79,12 @@ from .kubernetes import (
|
||||
render_kubernetes,
|
||||
write_secret_creation_hint,
|
||||
)
|
||||
from .planning import DeploymentPlan, build_plan
|
||||
from .planning import (
|
||||
DeploymentPlan,
|
||||
build_plan,
|
||||
release_change_requires_backup,
|
||||
verify_stored_backup_evidence,
|
||||
)
|
||||
from .recovery import (
|
||||
DeploymentOperationJournal,
|
||||
list_operations,
|
||||
@@ -183,6 +195,22 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="Load verified archives into Docker using fixed image-load commands.",
|
||||
)
|
||||
|
||||
verify_backup = subparsers.add_parser(
|
||||
"verify-backup",
|
||||
help="Verify and optionally adopt signed coordinated backup evidence.",
|
||||
)
|
||||
_directory_argument(verify_backup)
|
||||
evidence_source = verify_backup.add_mutually_exclusive_group(required=True)
|
||||
evidence_source.add_argument("--evidence", type=Path)
|
||||
evidence_source.add_argument("--evidence-url")
|
||||
verify_backup.add_argument("--evidence-sha256", required=True)
|
||||
verify_backup.add_argument("--trusted-keyring", type=Path, required=True)
|
||||
verify_backup.add_argument(
|
||||
"--allow-private-evidence-host",
|
||||
action="store_true",
|
||||
)
|
||||
verify_backup.add_argument("--adopt", action="store_true")
|
||||
|
||||
kubernetes = subparsers.add_parser(
|
||||
"render-kubernetes",
|
||||
help="Export the stateless multi-host runtime for Kubernetes.",
|
||||
@@ -392,6 +420,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
return _verify_release(args)
|
||||
if args.command == "verify-offline-images":
|
||||
return _verify_offline_images(args)
|
||||
if args.command == "verify-backup":
|
||||
return _verify_backup(args)
|
||||
if args.command == "render-kubernetes":
|
||||
return _render_kubernetes(args)
|
||||
if args.command == "verify-kubernetes":
|
||||
@@ -534,15 +564,17 @@ def _render_or_doctor(args: argparse.Namespace) -> int:
|
||||
|
||||
def _apply(args: argparse.Namespace) -> int:
|
||||
paths = bundle_paths(args.directory)
|
||||
spec = load_spec(paths.spec)
|
||||
if args.allow_unverified_images and spec.profile != "evaluation":
|
||||
raise ValueError(
|
||||
"--allow-unverified-images is restricted to evaluation installations"
|
||||
)
|
||||
ensure_private_directory(paths.root)
|
||||
with _deployment_lock(paths.lock):
|
||||
spec = load_spec(paths.spec)
|
||||
previous_receipt = _read_json_object(paths.receipt)
|
||||
backup_required = release_change_requires_backup(spec, previous_receipt)
|
||||
if args.allow_unverified_images and spec.profile != "evaluation":
|
||||
raise ValueError(
|
||||
"--allow-unverified-images is restricted to evaluation installations"
|
||||
)
|
||||
secrets = reconcile_runtime_environment(spec, read_env(paths.env))
|
||||
_write_bundle(spec, paths, secrets)
|
||||
secrets = _write_bundle(spec, paths, secrets)
|
||||
plan = build_plan(spec, paths, include_host_checks=True)
|
||||
_write_plan(paths.plan, plan)
|
||||
effective_errors = [
|
||||
@@ -577,6 +609,36 @@ def _apply(args: argparse.Namespace) -> int:
|
||||
paths,
|
||||
plan=plan.to_dict(),
|
||||
)
|
||||
try:
|
||||
if backup_required:
|
||||
backup_summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=previous_receipt,
|
||||
)
|
||||
journal.record(
|
||||
"backup-evidence-verified",
|
||||
"succeeded",
|
||||
dict(backup_summary),
|
||||
)
|
||||
else:
|
||||
journal.record(
|
||||
"backup-evidence-not-required",
|
||||
"succeeded",
|
||||
{"release_change": False},
|
||||
)
|
||||
except BaseException as exc:
|
||||
journal.record(
|
||||
"backup-evidence-rejected",
|
||||
"blocked",
|
||||
{
|
||||
"phase": "preflight",
|
||||
"exception_type": type(exc).__name__,
|
||||
"migration_started": False,
|
||||
},
|
||||
)
|
||||
journal.failed(exc)
|
||||
raise
|
||||
compose = [
|
||||
docker,
|
||||
"compose",
|
||||
@@ -624,6 +686,29 @@ def _apply(args: argparse.Namespace) -> int:
|
||||
"succeeded",
|
||||
{"services": mutable_runtime_services, "timeout_seconds": 120},
|
||||
)
|
||||
if backup_required:
|
||||
try:
|
||||
backup_summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=previous_receipt,
|
||||
)
|
||||
except BaseException as exc:
|
||||
journal.record(
|
||||
"backup-evidence-rejected",
|
||||
"blocked",
|
||||
{
|
||||
"phase": "migration-boundary",
|
||||
"exception_type": type(exc).__name__,
|
||||
"migration_started": False,
|
||||
},
|
||||
)
|
||||
raise
|
||||
journal.record(
|
||||
"backup-evidence-reverified",
|
||||
"succeeded",
|
||||
dict(backup_summary),
|
||||
)
|
||||
journal.migration_started()
|
||||
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
|
||||
journal.migration_completed()
|
||||
@@ -898,6 +983,108 @@ def _verify_offline_images(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _verify_backup(args: argparse.Namespace) -> int:
|
||||
paths = bundle_paths(args.directory)
|
||||
ensure_private_directory(paths.root)
|
||||
with _deployment_lock(paths.lock):
|
||||
return _verify_backup_locked(args, paths)
|
||||
|
||||
|
||||
def _verify_backup_locked(args: argparse.Namespace, paths) -> int:
|
||||
spec = load_spec(paths.spec)
|
||||
expected_digest = str(args.evidence_sha256 or "").strip().lower()
|
||||
if len(expected_digest) != 64 or any(
|
||||
character not in "0123456789abcdef" for character in expected_digest
|
||||
):
|
||||
raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest")
|
||||
if args.evidence_url:
|
||||
encoded_evidence = fetch_bounded_https(
|
||||
str(args.evidence_url),
|
||||
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
||||
allow_private_host=args.allow_private_evidence_host,
|
||||
)
|
||||
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
||||
else:
|
||||
evidence_path = args.evidence.expanduser().resolve()
|
||||
encoded_evidence = read_bounded_bytes(
|
||||
evidence_path,
|
||||
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
||||
)
|
||||
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
||||
if encoded_evidence != canonical_distribution_json(evidence):
|
||||
raise DistributionError("backup evidence is not canonical JSON")
|
||||
if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest:
|
||||
raise DistributionError("backup evidence SHA-256 does not match")
|
||||
keyring_path = args.trusted_keyring.expanduser().resolve()
|
||||
keyring = load_bounded_json(
|
||||
keyring_path,
|
||||
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
|
||||
)
|
||||
encoded_keyring = canonical_distribution_json(keyring)
|
||||
receipt = _read_json_object(paths.receipt)
|
||||
previous_release = receipt.get("release") if receipt else None
|
||||
release: Mapping[str, object] = (
|
||||
previous_release
|
||||
if isinstance(previous_release, Mapping)
|
||||
else {
|
||||
"channel": spec.release.channel,
|
||||
"version": spec.release.version,
|
||||
"manifest_sha256": spec.release.manifest_sha256,
|
||||
"composition_sha256": spec.release.composition_sha256,
|
||||
"api_image": spec.release.api_image,
|
||||
"web_image": spec.release.web_image,
|
||||
}
|
||||
)
|
||||
summary = verify_backup_evidence(
|
||||
evidence,
|
||||
keyring,
|
||||
installation_id=spec.installation_id,
|
||||
profile=spec.profile,
|
||||
release=release,
|
||||
max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS,
|
||||
)
|
||||
print(
|
||||
"Verified coordinated recovery point "
|
||||
f"{summary['recovery_point_id']} with restore drill "
|
||||
f"{summary['restore_drill_id']} and trusted key "
|
||||
f"{summary['signature_key_id']}."
|
||||
)
|
||||
if not args.adopt:
|
||||
return 0
|
||||
verification = {
|
||||
"schema_version": 1,
|
||||
"evidence_sha256": expected_digest,
|
||||
"keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(),
|
||||
"signature_key_id": summary["signature_key_id"],
|
||||
"verified_at": _now(),
|
||||
"evidence_id": summary["evidence_id"],
|
||||
"recovery_point_id": summary["recovery_point_id"],
|
||||
"restore_drill_id": summary["restore_drill_id"],
|
||||
"release_manifest_sha256": release.get("manifest_sha256"),
|
||||
"captured_at": summary["captured_at"],
|
||||
"expires_at": summary["expires_at"],
|
||||
"restore_started_at": summary["restore_started_at"],
|
||||
"restore_completed_at": summary["restore_completed_at"],
|
||||
"measured_rpo_seconds": summary["measured_rpo_seconds"],
|
||||
"measured_rto_seconds": summary["measured_rto_seconds"],
|
||||
"component_count": summary["component_count"],
|
||||
}
|
||||
atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600)
|
||||
atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600)
|
||||
atomic_write(
|
||||
paths.backup_verification,
|
||||
canonical_json(verification),
|
||||
mode=0o600,
|
||||
)
|
||||
_write_bundle(
|
||||
spec,
|
||||
paths,
|
||||
reconcile_runtime_environment(spec, read_env(paths.env)),
|
||||
)
|
||||
print(f"Adopted signed backup evidence in {paths.root}.")
|
||||
return 0
|
||||
|
||||
|
||||
def _selected_dependency_images(
|
||||
spec: InstallationSpec,
|
||||
*,
|
||||
@@ -929,6 +1116,30 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
|
||||
paths = bundle_paths(args.directory)
|
||||
spec = load_spec(paths.spec)
|
||||
environment = reconcile_runtime_environment(spec, read_env(paths.env))
|
||||
environment.update(_backup_runtime_environment(spec, paths))
|
||||
receipt = _read_json_object(paths.receipt)
|
||||
backup_required = release_change_requires_backup(spec, receipt)
|
||||
backup_summary: Mapping[str, object] | None = None
|
||||
evidence_files = (
|
||||
paths.backup_evidence,
|
||||
paths.backup_keyring,
|
||||
paths.backup_verification,
|
||||
)
|
||||
if backup_required:
|
||||
backup_summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=receipt,
|
||||
)
|
||||
elif all(path.is_file() for path in evidence_files):
|
||||
try:
|
||||
backup_summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=receipt,
|
||||
)
|
||||
except (DistributionError, OSError):
|
||||
backup_summary = None
|
||||
manifest = render_kubernetes(
|
||||
spec,
|
||||
environment,
|
||||
@@ -936,6 +1147,8 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
|
||||
secret_name=args.secret_name,
|
||||
tls_secret_name=args.tls_secret_name,
|
||||
ingress_class_name=args.ingress_class_name,
|
||||
backup_required=backup_required,
|
||||
backup_evidence=backup_summary,
|
||||
)
|
||||
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
|
||||
atomic_write(output, canonical_json(manifest), mode=0o600)
|
||||
@@ -1027,6 +1240,7 @@ def _deployment_receipt(
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"installation_id": spec.installation_id,
|
||||
"profile": spec.profile,
|
||||
"applied_at": _now(),
|
||||
"spec_sha256": digest_json(spec.to_dict()),
|
||||
"compose_sha256": digest_json(render_compose(spec)),
|
||||
@@ -1064,6 +1278,16 @@ def _deployment_receipt(
|
||||
}
|
||||
|
||||
|
||||
def _read_json_object(path: Path) -> dict[str, object]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
value = load_bounded_json(path, maximum_bytes=64 * 1024)
|
||||
except (DistributionError, OSError):
|
||||
return {}
|
||||
return value
|
||||
|
||||
|
||||
def _updated_spec(
|
||||
current: InstallationSpec, args: argparse.Namespace
|
||||
) -> InstallationSpec:
|
||||
@@ -1222,10 +1446,12 @@ def _write_bundle(
|
||||
spec: InstallationSpec,
|
||||
paths,
|
||||
secrets: Mapping[str, str],
|
||||
) -> None:
|
||||
) -> dict[str, str]:
|
||||
ensure_private_directory(paths.root)
|
||||
runtime_environment = dict(secrets)
|
||||
runtime_environment.update(_backup_runtime_environment(spec, paths))
|
||||
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
|
||||
write_env(paths.env, secrets)
|
||||
write_env(paths.env, runtime_environment)
|
||||
atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
|
||||
atomic_write(
|
||||
paths.load_balancer_config,
|
||||
@@ -1247,6 +1473,62 @@ def _write_bundle(
|
||||
render_garage_config().encode("utf-8"),
|
||||
mode=0o644,
|
||||
)
|
||||
return dict(sorted(runtime_environment.items()))
|
||||
|
||||
|
||||
def _backup_runtime_environment(
|
||||
spec: InstallationSpec,
|
||||
paths,
|
||||
) -> dict[str, str]:
|
||||
values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS}
|
||||
evidence_files = (
|
||||
paths.backup_evidence,
|
||||
paths.backup_keyring,
|
||||
paths.backup_verification,
|
||||
)
|
||||
if not any(path.is_file() for path in evidence_files):
|
||||
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent"
|
||||
return values
|
||||
if not all(path.is_file() for path in evidence_files):
|
||||
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
|
||||
return values
|
||||
verification = _read_json_object(paths.backup_verification)
|
||||
try:
|
||||
summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=_read_json_object(paths.receipt),
|
||||
)
|
||||
except (DistributionError, OSError):
|
||||
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
|
||||
return values
|
||||
values.update(
|
||||
{
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]),
|
||||
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]),
|
||||
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]),
|
||||
"GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]),
|
||||
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str(
|
||||
verification["release_manifest_sha256"]
|
||||
),
|
||||
"GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]),
|
||||
"GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]),
|
||||
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]),
|
||||
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str(
|
||||
summary["restore_completed_at"]
|
||||
),
|
||||
"GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]),
|
||||
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str(
|
||||
summary["measured_rpo_seconds"]
|
||||
),
|
||||
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str(
|
||||
summary["measured_rto_seconds"]
|
||||
),
|
||||
"GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]),
|
||||
}
|
||||
)
|
||||
return values
|
||||
|
||||
|
||||
def _write_plan(path: Path, plan: DeploymentPlan) -> None:
|
||||
|
||||
@@ -74,7 +74,9 @@ def read_bounded_bytes(path: Path, *, maximum_bytes: int) -> bytes:
|
||||
try:
|
||||
opened = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
|
||||
raise DistributionError(f"trusted JSON file is invalid or too large: {path}")
|
||||
raise DistributionError(
|
||||
f"trusted JSON file is invalid or too large: {path}"
|
||||
)
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
@@ -109,7 +111,9 @@ def fetch_bounded_https(
|
||||
if parsed.scheme != "https" or not parsed.hostname:
|
||||
raise DistributionError("distribution downloads require an absolute HTTPS URL")
|
||||
if parsed.username or parsed.password or parsed.fragment:
|
||||
raise DistributionError("distribution URL must not contain credentials or a fragment")
|
||||
raise DistributionError(
|
||||
"distribution URL must not contain credentials or a fragment"
|
||||
)
|
||||
if not allow_private_host:
|
||||
_require_public_host(parsed.hostname)
|
||||
request = Request(url, headers={"Accept": "application/json"})
|
||||
@@ -172,9 +176,11 @@ def validate_manifest(
|
||||
raise DistributionError(
|
||||
f"distribution channel is {channel!r}, expected {expected_channel!r}"
|
||||
)
|
||||
if isinstance(payload.get("sequence"), bool) or not isinstance(
|
||||
payload.get("sequence"), int
|
||||
) or int(payload["sequence"]) < 1:
|
||||
if (
|
||||
isinstance(payload.get("sequence"), bool)
|
||||
or not isinstance(payload.get("sequence"), int)
|
||||
or int(payload["sequence"]) < 1
|
||||
):
|
||||
raise DistributionError("distribution sequence must be a positive integer")
|
||||
_token(payload.get("version"), "version", maximum=128, pattern=TOKEN)
|
||||
issued = _datetime(payload.get("issued_at"), "issued_at")
|
||||
@@ -283,11 +289,41 @@ def verify_manifest(
|
||||
) -> str:
|
||||
current = (now or datetime.now(UTC)).astimezone(UTC)
|
||||
validate_manifest(payload, expected_channel=expected_channel, now=current)
|
||||
keys = _trusted_keys(keyring, now=current)
|
||||
return verify_signed_document(
|
||||
payload,
|
||||
keyring,
|
||||
purpose="govoplan-runtime-distribution",
|
||||
label="distribution",
|
||||
now=current,
|
||||
openssl=openssl,
|
||||
)
|
||||
|
||||
|
||||
def verify_signed_document(
|
||||
payload: Mapping[str, Any],
|
||||
keyring: Mapping[str, Any],
|
||||
*,
|
||||
purpose: str,
|
||||
label: str,
|
||||
now: datetime,
|
||||
openssl: str = "openssl",
|
||||
) -> str:
|
||||
keys = _trusted_keys(keyring, now=now, purpose=purpose, label=label)
|
||||
signed = canonical_signed_payload(payload)
|
||||
failures: list[str] = []
|
||||
for item in payload["signatures"]:
|
||||
signatures = payload.get("signatures")
|
||||
if not isinstance(signatures, list) or not signatures:
|
||||
raise DistributionError(f"{label} has no signatures")
|
||||
for index, raw in enumerate(signatures):
|
||||
item = _object(raw, f"{label}.signatures[{index}]")
|
||||
_exact_keys(
|
||||
item,
|
||||
required={"key_id", "algorithm", "value"},
|
||||
label=f"{label}.signatures[{index}]",
|
||||
)
|
||||
key_id = str(item["key_id"])
|
||||
if KEY_ID.fullmatch(key_id) is None or item.get("algorithm") != "ed25519":
|
||||
raise DistributionError(f"{label} signature is invalid")
|
||||
public_key = keys.get(key_id)
|
||||
if public_key is None:
|
||||
continue
|
||||
@@ -303,8 +339,10 @@ def verify_manifest(
|
||||
failures.append(f"{key_id}: {exc}")
|
||||
continue
|
||||
return key_id
|
||||
detail = "; ".join(failures) if failures else "no signature used an active trusted key"
|
||||
raise DistributionError(f"distribution signature verification failed: {detail}")
|
||||
detail = (
|
||||
"; ".join(failures) if failures else "no signature used an active trusted key"
|
||||
)
|
||||
raise DistributionError(f"{label} signature verification failed: {detail}")
|
||||
|
||||
|
||||
def verify_manifest_binding(
|
||||
@@ -319,7 +357,9 @@ def verify_manifest_binding(
|
||||
dependencies: Mapping[str, str],
|
||||
) -> None:
|
||||
if payload.get("channel") != channel or payload.get("version") != version:
|
||||
raise DistributionError("stored manifest does not match release channel/version")
|
||||
raise DistributionError(
|
||||
"stored manifest does not match release channel/version"
|
||||
)
|
||||
images = _object(payload.get("images"), "images")
|
||||
if _object(images.get("api"), "images.api").get("index") != api_image:
|
||||
raise DistributionError("stored manifest does not match API image")
|
||||
@@ -372,9 +412,9 @@ def verify_offline_image_index(
|
||||
archive = root / archive_relative
|
||||
if reference in references:
|
||||
raise DistributionError("offline image index contains duplicate references")
|
||||
if _sha256_regular_file(archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES) != _sha256(
|
||||
value.get("sha256"), "offline image sha256"
|
||||
):
|
||||
if _sha256_regular_file(
|
||||
archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES
|
||||
) != _sha256(value.get("sha256"), "offline image sha256"):
|
||||
raise DistributionError(f"offline image archive digest mismatch: {archive}")
|
||||
references[reference] = archive
|
||||
missing = sorted(set(expected_references) - set(references))
|
||||
@@ -418,19 +458,21 @@ def _trusted_keys(
|
||||
keyring: Mapping[str, Any],
|
||||
*,
|
||||
now: datetime,
|
||||
purpose: str,
|
||||
label: str,
|
||||
) -> dict[str, str]:
|
||||
_exact_keys(
|
||||
keyring,
|
||||
required={"schema_version", "purpose", "keys"},
|
||||
label="distribution keyring",
|
||||
label=f"{label} keyring",
|
||||
)
|
||||
if keyring.get("schema_version") != "1":
|
||||
raise DistributionError("unsupported distribution keyring schema_version")
|
||||
if keyring.get("purpose") != "govoplan-runtime-distribution":
|
||||
raise DistributionError("distribution keyring has the wrong purpose")
|
||||
raise DistributionError(f"unsupported {label} keyring schema_version")
|
||||
if keyring.get("purpose") != purpose:
|
||||
raise DistributionError(f"{label} keyring has the wrong purpose")
|
||||
values = keyring.get("keys")
|
||||
if not isinstance(values, list) or not values:
|
||||
raise DistributionError("distribution keyring contains no keys")
|
||||
raise DistributionError(f"{label} keyring contains no keys")
|
||||
trusted: dict[str, str] = {}
|
||||
for index, item in enumerate(values):
|
||||
key = _object(item, f"keyring.keys[{index}]")
|
||||
@@ -453,11 +495,11 @@ def _trusted_keys(
|
||||
pattern=KEY_ID,
|
||||
)
|
||||
if key_id in trusted:
|
||||
raise DistributionError("distribution keyring contains duplicate key ids")
|
||||
raise DistributionError(f"{label} keyring contains duplicate key ids")
|
||||
if key.get("algorithm") != "ed25519":
|
||||
raise DistributionError("distribution key must use ed25519")
|
||||
raise DistributionError(f"{label} key must use ed25519")
|
||||
if key.get("status") not in {"active", "retired", "revoked"}:
|
||||
raise DistributionError("distribution key has an invalid status")
|
||||
raise DistributionError(f"{label} key has an invalid status")
|
||||
not_before = _datetime(key.get("not_before"), "key.not_before")
|
||||
expires = _datetime(key.get("expires_at"), "key.expires_at")
|
||||
public_key = key.get("public_key_pem")
|
||||
@@ -466,11 +508,11 @@ def _trusted_keys(
|
||||
or len(public_key.encode("utf-8")) > 8192
|
||||
or "BEGIN PUBLIC KEY" not in public_key
|
||||
):
|
||||
raise DistributionError("distribution key has an invalid public key")
|
||||
raise DistributionError(f"{label} key has an invalid public key")
|
||||
if key.get("status") == "active" and not_before <= now < expires:
|
||||
trusted[key_id] = public_key
|
||||
if not trusted:
|
||||
raise DistributionError("distribution keyring has no currently active keys")
|
||||
raise DistributionError(f"{label} keyring has no currently active keys")
|
||||
return trusted
|
||||
|
||||
|
||||
@@ -534,13 +576,17 @@ def _require_public_host(hostname: str) -> None:
|
||||
for value in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM)
|
||||
}
|
||||
except OSError as exc:
|
||||
raise DistributionError(f"distribution host cannot be resolved: {hostname}") from exc
|
||||
raise DistributionError(
|
||||
f"distribution host cannot be resolved: {hostname}"
|
||||
) from exc
|
||||
if not addresses:
|
||||
raise DistributionError("distribution host resolved to no addresses")
|
||||
for value in addresses:
|
||||
address = ipaddress.ip_address(value)
|
||||
if not address.is_global:
|
||||
raise DistributionError("distribution host resolves to a non-public address")
|
||||
raise DistributionError(
|
||||
"distribution host resolves to a non-public address"
|
||||
)
|
||||
|
||||
|
||||
def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
|
||||
@@ -553,7 +599,9 @@ def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
|
||||
try:
|
||||
opened = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
|
||||
raise DistributionError(f"immutable artifact is invalid or too large: {path}")
|
||||
raise DistributionError(
|
||||
f"immutable artifact is invalid or too large: {path}"
|
||||
)
|
||||
while True:
|
||||
chunk = os.read(descriptor, 1024 * 1024)
|
||||
if not chunk:
|
||||
@@ -602,7 +650,11 @@ def _token(
|
||||
maximum: int,
|
||||
pattern: re.Pattern[str],
|
||||
) -> str:
|
||||
if not isinstance(value, str) or len(value) > maximum or pattern.fullmatch(value) is None:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or len(value) > maximum
|
||||
or pattern.fullmatch(value) is None
|
||||
):
|
||||
raise DistributionError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
@@ -626,7 +678,11 @@ def _sha256(value: object, label: str) -> str:
|
||||
|
||||
|
||||
def _digest_image(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or len(value) > 300 or DIGEST_IMAGE.fullmatch(value) is None:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or len(value) > 300
|
||||
or DIGEST_IMAGE.fullmatch(value) is None
|
||||
):
|
||||
raise DistributionError(f"{label} must be an OCI image pinned by sha256")
|
||||
return value
|
||||
|
||||
@@ -635,7 +691,12 @@ def _https_url(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or len(value) > 2048:
|
||||
raise DistributionError(f"{label} must be an HTTPS URL")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or not parsed.netloc
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
):
|
||||
raise DistributionError(f"{label} must be an HTTPS URL without credentials")
|
||||
return value
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import re
|
||||
from typing import Any, Mapping
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .bundle import BACKUP_RUNTIME_ENV_KEYS
|
||||
from .model import InstallationSpec, image_is_digest_pinned
|
||||
|
||||
|
||||
@@ -55,6 +56,7 @@ _CONFIG_KEYS = (
|
||||
"FILE_STORAGE_S3_BUCKET",
|
||||
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
||||
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
||||
*BACKUP_RUNTIME_ENV_KEYS,
|
||||
)
|
||||
_QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
|
||||
|
||||
@@ -75,6 +77,8 @@ def render_kubernetes(
|
||||
secret_name: str = "govoplan-runtime",
|
||||
tls_secret_name: str = "govoplan-tls",
|
||||
ingress_class_name: str | None = None,
|
||||
backup_required: bool = True,
|
||||
backup_evidence: Mapping[str, object] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Render runtime roles only; shared state services stay externally managed."""
|
||||
|
||||
@@ -199,6 +203,8 @@ def render_kubernetes(
|
||||
environment,
|
||||
"MIGRATION",
|
||||
),
|
||||
backup_required=backup_required,
|
||||
backup_evidence=backup_evidence,
|
||||
),
|
||||
]
|
||||
for pool in worker_pools:
|
||||
@@ -765,9 +771,28 @@ def _migration_job(
|
||||
secret_name: str,
|
||||
service_account: str,
|
||||
database_environment: Mapping[str, str],
|
||||
backup_required: bool,
|
||||
backup_evidence: Mapping[str, object] | None,
|
||||
) -> dict[str, Any]:
|
||||
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
|
||||
job_name = _name_with_suffix(name, f"migrate-{release_key}")
|
||||
backup_annotations = {
|
||||
"govoplan.add-ideas.de/backup-required": str(backup_required).lower(),
|
||||
}
|
||||
if backup_evidence is not None:
|
||||
backup_annotations.update(
|
||||
{
|
||||
"govoplan.add-ideas.de/backup-evidence-sha256": str(
|
||||
backup_evidence["evidence_sha256"]
|
||||
),
|
||||
"govoplan.add-ideas.de/recovery-point": str(
|
||||
backup_evidence["recovery_point_id"]
|
||||
),
|
||||
"govoplan.add-ideas.de/restore-drill": str(
|
||||
backup_evidence["restore_drill_id"]
|
||||
),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
@@ -777,7 +802,7 @@ def _migration_job(
|
||||
"labels": job_labels,
|
||||
"annotations": {
|
||||
"govoplan.add-ideas.de/recovery-mode": "forward-recovery",
|
||||
"govoplan.add-ideas.de/backup-required": "true",
|
||||
**backup_annotations,
|
||||
"argocd.argoproj.io/sync-wave": "-1",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -18,6 +19,11 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .backup_evidence import (
|
||||
MAX_BACKUP_EVIDENCE_BYTES,
|
||||
MAX_BACKUP_KEYRING_BYTES,
|
||||
verify_backup_evidence,
|
||||
)
|
||||
from .bundle import (
|
||||
BundlePaths,
|
||||
canonical_json,
|
||||
@@ -33,8 +39,11 @@ from .distribution import (
|
||||
MAX_KEYRING_BYTES,
|
||||
MAX_MANIFEST_BYTES,
|
||||
DistributionError,
|
||||
canonical_json as canonical_distribution_json,
|
||||
decode_json_bytes,
|
||||
file_sha256,
|
||||
load_bounded_json,
|
||||
read_bounded_bytes,
|
||||
verify_manifest,
|
||||
verify_manifest_binding,
|
||||
)
|
||||
@@ -232,6 +241,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
|
||||
)
|
||||
|
||||
checks.extend(_distribution_checks(spec, paths))
|
||||
checks.extend(
|
||||
_backup_evidence_checks(spec, paths, receipt=_read_receipt(paths.receipt))
|
||||
)
|
||||
|
||||
values = read_env(paths.env)
|
||||
required = {"MASTER_KEY_B64", "DATABASE_URL"}
|
||||
@@ -350,6 +362,189 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
|
||||
return tuple(checks)
|
||||
|
||||
|
||||
def release_change_requires_backup(
|
||||
spec: InstallationSpec,
|
||||
receipt: Mapping[str, object],
|
||||
) -> bool:
|
||||
if spec.profile != "self-hosted" or not receipt:
|
||||
return False
|
||||
previous = receipt.get("release")
|
||||
if not isinstance(previous, Mapping):
|
||||
return True
|
||||
desired = {
|
||||
"channel": spec.release.channel,
|
||||
"version": spec.release.version,
|
||||
"manifest_sha256": spec.release.manifest_sha256,
|
||||
"composition_sha256": spec.release.composition_sha256,
|
||||
"api_image": spec.release.api_image,
|
||||
"web_image": spec.release.web_image,
|
||||
}
|
||||
return any(previous.get(key) != value for key, value in desired.items())
|
||||
|
||||
|
||||
def verify_stored_backup_evidence(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
receipt: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
verification = load_bounded_json(
|
||||
paths.backup_verification,
|
||||
maximum_bytes=64 * 1024,
|
||||
)
|
||||
expected_fields = {
|
||||
"schema_version",
|
||||
"evidence_sha256",
|
||||
"keyring_sha256",
|
||||
"signature_key_id",
|
||||
"verified_at",
|
||||
"evidence_id",
|
||||
"recovery_point_id",
|
||||
"restore_drill_id",
|
||||
"release_manifest_sha256",
|
||||
"captured_at",
|
||||
"expires_at",
|
||||
"restore_started_at",
|
||||
"restore_completed_at",
|
||||
"measured_rpo_seconds",
|
||||
"measured_rto_seconds",
|
||||
"component_count",
|
||||
}
|
||||
if set(verification) != expected_fields or verification.get("schema_version") != 1:
|
||||
raise DistributionError("backup verification receipt is malformed")
|
||||
encoded_evidence = read_bounded_bytes(
|
||||
paths.backup_evidence,
|
||||
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
||||
)
|
||||
encoded_keyring = read_bounded_bytes(
|
||||
paths.backup_keyring,
|
||||
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
|
||||
)
|
||||
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
||||
keyring = decode_json_bytes(encoded_keyring, label="backup keyring")
|
||||
if encoded_evidence != canonical_distribution_json(evidence):
|
||||
raise DistributionError("stored backup evidence is not canonical JSON")
|
||||
if encoded_keyring != canonical_distribution_json(keyring):
|
||||
raise DistributionError("stored backup keyring is not canonical JSON")
|
||||
evidence_digest = hashlib.sha256(encoded_evidence).hexdigest()
|
||||
keyring_digest = hashlib.sha256(encoded_keyring).hexdigest()
|
||||
if evidence_digest != verification.get("evidence_sha256"):
|
||||
raise DistributionError("stored backup evidence digest has changed")
|
||||
if keyring_digest != verification.get("keyring_sha256"):
|
||||
raise DistributionError("stored backup keyring digest has changed")
|
||||
previous_release = receipt.get("release") if receipt else None
|
||||
expected_release: Mapping[str, object] = (
|
||||
previous_release
|
||||
if isinstance(previous_release, Mapping)
|
||||
else {
|
||||
"channel": spec.release.channel,
|
||||
"version": spec.release.version,
|
||||
"manifest_sha256": spec.release.manifest_sha256,
|
||||
"composition_sha256": spec.release.composition_sha256,
|
||||
"api_image": spec.release.api_image,
|
||||
"web_image": spec.release.web_image,
|
||||
}
|
||||
)
|
||||
summary = verify_backup_evidence(
|
||||
evidence,
|
||||
keyring,
|
||||
installation_id=spec.installation_id,
|
||||
profile=spec.profile,
|
||||
release=expected_release,
|
||||
)
|
||||
expected_summary = {
|
||||
"signature_key_id": verification.get("signature_key_id"),
|
||||
"evidence_id": verification.get("evidence_id"),
|
||||
"recovery_point_id": verification.get("recovery_point_id"),
|
||||
"restore_drill_id": verification.get("restore_drill_id"),
|
||||
"captured_at": verification.get("captured_at"),
|
||||
"expires_at": verification.get("expires_at"),
|
||||
"restore_started_at": verification.get("restore_started_at"),
|
||||
"restore_completed_at": verification.get("restore_completed_at"),
|
||||
"measured_rpo_seconds": verification.get("measured_rpo_seconds"),
|
||||
"measured_rto_seconds": verification.get("measured_rto_seconds"),
|
||||
"component_count": verification.get("component_count"),
|
||||
}
|
||||
for field, expected in expected_summary.items():
|
||||
if summary.get(field) != expected:
|
||||
raise DistributionError(
|
||||
f"backup verification receipt does not match {field!r}"
|
||||
)
|
||||
if expected_release.get("manifest_sha256") != verification.get(
|
||||
"release_manifest_sha256"
|
||||
):
|
||||
raise DistributionError("backup verification receipt has another release")
|
||||
return {
|
||||
**summary,
|
||||
"evidence_sha256": evidence_digest,
|
||||
"keyring_sha256": keyring_digest,
|
||||
}
|
||||
|
||||
|
||||
def _backup_evidence_checks(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
*,
|
||||
receipt: Mapping[str, object],
|
||||
) -> tuple[Check, ...]:
|
||||
required = release_change_requires_backup(spec, receipt)
|
||||
available = all(
|
||||
path.is_file()
|
||||
for path in (
|
||||
paths.backup_evidence,
|
||||
paths.backup_keyring,
|
||||
paths.backup_verification,
|
||||
)
|
||||
)
|
||||
if not available:
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"error"
|
||||
if required
|
||||
else "warning"
|
||||
if spec.profile == "self-hosted"
|
||||
else "ok",
|
||||
(
|
||||
"A release-changing migration has no verified coordinated backup evidence."
|
||||
if required
|
||||
else "No current coordinated backup evidence is adopted."
|
||||
),
|
||||
(
|
||||
"Run verify-backup --adopt after an isolated restore drill."
|
||||
if spec.profile == "self-hosted"
|
||||
else ""
|
||||
),
|
||||
),
|
||||
)
|
||||
try:
|
||||
summary = verify_stored_backup_evidence(
|
||||
spec,
|
||||
paths,
|
||||
receipt=receipt,
|
||||
)
|
||||
except (DistributionError, OSError) as exc:
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"error" if required else "warning",
|
||||
f"Coordinated backup evidence is invalid: {exc}",
|
||||
"Adopt fresh signed evidence for the currently applied release.",
|
||||
),
|
||||
)
|
||||
return (
|
||||
Check(
|
||||
"backup.migration_gate",
|
||||
"ok",
|
||||
(
|
||||
"Release migration is backed by recovery point "
|
||||
f"{summary['recovery_point_id']} and restore drill "
|
||||
f"{summary['restore_drill_id']}."
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _ingress_configuration_checks(
|
||||
spec: InstallationSpec,
|
||||
paths: BundlePaths,
|
||||
|
||||
@@ -29,6 +29,9 @@ _BUNDLE_FILES = (
|
||||
"existing-proxy.json",
|
||||
"distribution-manifest.json",
|
||||
"distribution-keyring.json",
|
||||
"backup-evidence.json",
|
||||
"backup-keyring.json",
|
||||
"backup-verification.json",
|
||||
"receipt.json",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user