From cbbe08d91256ec32000d4183649cda00ede9a6d1 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Mon, 3 Aug 2026 02:01:13 +0200 Subject: [PATCH] Enforce signed backup evidence before migrations --- docs/BACKUP_AND_RESTORE_EVIDENCE.md | 133 +++++ ...NSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md | 17 +- docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md | 14 +- docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md | 6 +- docs/backup-evidence-keyring.schema.json | 37 ++ docs/backup-evidence.schema.json | 255 +++++++++ tests/test_backup_evidence.py | 430 +++++++++++++++ tests/test_deployment_installer.py | 23 +- .../govoplan_deploy/backup_evidence.py | 500 ++++++++++++++++++ tools/deployment/govoplan_deploy/bundle.py | 26 + tools/deployment/govoplan_deploy/cli.py | 300 ++++++++++- .../govoplan_deploy/distribution.py | 119 ++++- .../deployment/govoplan_deploy/kubernetes.py | 27 +- tools/deployment/govoplan_deploy/planning.py | 195 +++++++ tools/deployment/govoplan_deploy/recovery.py | 3 + tools/deployment/sign-backup-evidence.py | 124 +++++ 16 files changed, 2157 insertions(+), 52 deletions(-) create mode 100644 docs/BACKUP_AND_RESTORE_EVIDENCE.md create mode 100644 docs/backup-evidence-keyring.schema.json create mode 100644 docs/backup-evidence.schema.json create mode 100644 tests/test_backup_evidence.py create mode 100644 tools/deployment/govoplan_deploy/backup_evidence.py create mode 100644 tools/deployment/sign-backup-evidence.py diff --git a/docs/BACKUP_AND_RESTORE_EVIDENCE.md b/docs/BACKUP_AND_RESTORE_EVIDENCE.md new file mode 100644 index 0000000..7ddeae3 --- /dev/null +++ b/docs/BACKUP_AND_RESTORE_EVIDENCE.md @@ -0,0 +1,133 @@ +# Backup And Restore Evidence + +## Boundary + +`govoplan-deploy` verifies backup and restore evidence; it does not receive +database, object-store, KMS, or orchestrator administration credentials and it +does not create the backup. A provider-owned backup controller creates one +coordinated recovery point, a separate drill runner restores it into an +isolated target, and an evidence authority signs the resulting receipt. + +The application containers receive only a sanitized projection: evidence, +recovery-point and drill identifiers, hashes, timestamps, component count, and +measured RPO/RTO. Artifact locations, provider credentials, encryption-key +references, the public trust keyring, and private signing keys remain in the +deployment/evidence boundary. + +The machine-readable contracts are: + +- [`backup-evidence.schema.json`](backup-evidence.schema.json); +- [`backup-evidence-keyring.schema.json`](backup-evidence-keyring.schema.json). + +One evidence document is bound to the installation id, deployment profile, +topology subject, exact signed release manifest, image digests, and composition +digest. It covers PostgreSQL, objects, protected configuration, and recoverable +key custody at one recovery point. It contains references, never key material. + +## Production Sequence + +1. Establish the provider snapshot, application quiesce, or transaction + boundary and retain a hash of its fencing token. +2. Capture PostgreSQL, object storage, protected deployment configuration, and + key-custody state within five minutes of that recovery point. +3. Restore all four components into a target isolated from production write + endpoints and production queues. +4. Start the exact immutable release named in the evidence, verify migration + heads, verify a deterministic manifest of representative object hashes, and + execute the documented semantic journey checks. +5. Record actual data loss and elapsed recovery as measured RPO and RTO. A + measured RPO above the declared objective invalidates the evidence. +6. Sign the canonical receipt using an evidence-authority Ed25519 key held + outside the application and deployment host. During key rotation, include + both accepted signatures. +7. Transfer the evidence SHA-256 through an independent approved channel, then + verify and adopt it on the deployment host. + +Provider automation can sign and validate an unsigned receipt with: + +```sh +python tools/deployment/sign-backup-evidence.py \ + --input unsigned-backup-evidence.json \ + --output backup-evidence.json \ + --trusted-keyring backup-evidence-keyring.json \ + --signing-key backup-authority-2026=/run/keys/backup-authority.pem +``` + +The private key file must be owner-only. The tool refuses an unexpected key +type, an inactive/untrusted signer, malformed or partial evidence, stale +recovery points, failed drill checks, mismatched releases, and non-canonical +output. + +Adopt the result using the independently obtained digest: + +```sh +python3 govoplan-deploy.pyz verify-backup \ + --directory /srv/govoplan/default \ + --evidence ./backup-evidence.json \ + --evidence-sha256 "$APPROVED_BACKUP_EVIDENCE_SHA256" \ + --trusted-keyring ./backup-evidence-keyring.json \ + --adopt +``` + +Evidence is fresh for at most 24 hours and may declare an earlier expiry. Every +self-hosted release identity change is conservatively treated as a migration +boundary. `doctor`, Compose `apply`, and `render-kubernetes` fail closed when +fresh evidence for the previously applied immutable release is unavailable. +Compose verifies once before changing runtime state and again after API/worker +quiescing immediately before migration. The exported Kubernetes migration Job +is generated only after verification and is annotated with the sanitized +evidence digest, recovery-point id, and drill id. + +## Provider Runbooks + +### PostgreSQL + +Use a managed transaction-consistent snapshot or a base backup plus retained +WAL sufficient to reconstruct the declared point. Record the provider, +protected artifact reference and digest, snapshot identity, and PostgreSQL LSN. +The restore drill must connect only to the isolated database and must compare +the resulting migration-head digest with the release expectation. + +### Object Storage + +Use provider snapshots/versioning or an immutable object copy. Build a sorted +manifest containing object key, version, size, and content digest, then record +its digest, object count, total bytes, provider version identity, and protected +artifact reference. Verify representative objects from every owning module +after restore. Single-node managed Garage is persistent but not highly +available; copy its coordinated recovery material to an independent failure +domain. + +### Configuration And Key Custody + +Back up the private installation bundle and external secret-manager bindings as +an encrypted artifact. Record only its reference and digest. For KMS/HSM/vault +state, record the provider keyset reference, version, and a successful +recoverability assertion. Never put a key, recovery share, token, password, or +credential-bearing URL in evidence. The isolated drill must prove that the +restored release can decrypt representative protected content without +exporting the key material into the report. + +## Ownership And Retention + +The deployment owner approves the RPO/RTO objectives. State-service owners +operate backup capture and restoration. Module owners define representative +objects and semantic checks. Security owns evidence-authority keys and +revocation. Operations schedules drills and retains sanitized status. + +Retain backup artifacts for the approved legal/operational period and at least +through the release's rollback window. Retain signed evidence, drill reports, +and deletion receipts for the audit period. Disposal must remove every backup +copy and provider version according to policy, then revoke or retire references +without deleting the audit receipt. Cryptographic erasure is valid only when +key-destruction evidence and provider-copy coverage are independently proven. + +## Failure Handling + +Missing components, component-time skew, stale or expired evidence, revocation, +signature/key mismatch, changed stored files, release mismatch, failed semantic +checks, or an RPO breach block migration. The deployment journal records the +rejection without private provider details. If migration has not started, the +operator may supply fresh evidence and retry. Once migration starts, recovery +is explicitly forward-only until the verified coordinated recovery point is +restored with its matching release. diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index 9eabe93..c56736d 100644 --- a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -97,6 +97,9 @@ The private installation directory contains: | `receipt.json` | Last successfully applied immutable identities | | `distribution-manifest.json` | Canonical signed runtime/image selection adopted by the installer | | `distribution-keyring.json` | Explicitly installed public trust anchor for runtime releases | +| `backup-evidence.json` | Signed provider-neutral coordinated backup and isolated-restore receipt | +| `backup-keyring.json` | Explicit public trust anchor for backup evidence authorities | +| `backup-verification.json` | Sanitized local verification/adoption receipt | | `applied-state/` | Checksum-verified snapshot of the last healthy deployment bundle | | `operations//` | Private hash-chained deployment progress and recovery evidence | | `kubernetes.json` | Optional stateless multi-host Kubernetes export | @@ -369,10 +372,12 @@ starts, recovery is forward-only unless an independently verified database backup is restored. See [Recovery And Rollback Guarantees](RECOVERY_AND_ROLLBACK_GUARANTEES.md). -Production updates still need an operator-provided database backup/restore -gate, database compatibility declaration, image signature verification, and -deployment-specific drain policy. The deployment journal proves its own -actions; it does not manufacture backup evidence. +Production updates still need operator/provider-created coordinated backup and +restore evidence, a database compatibility declaration, and a +deployment-specific drain policy. The deployer now verifies and enforces the +signed evidence before migration, but does not manufacture backups or receive +provider administration credentials. See +[Backup And Restore Evidence](BACKUP_AND_RESTORE_EVIDENCE.md). ## Stateless Kubernetes Runtime @@ -391,7 +396,9 @@ The output includes a release-specific migration Job, database-head wait init containers, API readiness/liveness probes, rolling Deployments, Services, Pod disruption budgets, a tokenless ServiceAccount, and one fenced scheduler. Apply the named Secret through the cluster's secret manager and review ingress proxy -CIDRs before deployment. Detailed rollout and scaling rules live in +CIDRs before deployment. A release-changing export requires adopted backup +evidence and carries only its sanitized digest and identifiers as Job +annotations. Detailed rollout and scaling rules live in [Scaling And Multi-Host Deployment](SCALING_AND_MULTI_HOST_DEPLOYMENT.md). ## Recovery Commands diff --git a/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md b/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md index d59ce75..b06f0e8 100644 --- a/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md +++ b/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md @@ -98,11 +98,15 @@ old code may not understand the new schema. Recovery then means one of: 3. restore a separately verified, coordinated database/object/key backup and then deploy the matching release. -The deployment tool does not create or validate that database backup. A -`backup-required` annotation on the Kubernetes migration Job is an operator -gate, not backup evidence. Production automation must provide a backup hook or -external backup controller whose artifact, timestamp, scope, encryption key, -and restore test can be referenced from the recovery record. +The deployment tool does not create that backup. It does verify an externally +produced, signed evidence contract covering PostgreSQL, objects, protected +configuration, and key custody at one recovery point plus an isolated restore +drill. A self-hosted release change cannot reach the migration command or be +exported as a Kubernetes migration Job until fresh evidence bound to the +previous immutable release has been adopted. Compose verifies it again after +runtime quiescing. See +[Backup And Restore Evidence](BACKUP_AND_RESTORE_EVIDENCE.md) for the contract, +provider runbooks, RPO/RTO ownership, retention, and disposal rules. ## Scaled Nodes diff --git a/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md index 01a27e4..775dfbc 100644 --- a/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md +++ b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md @@ -190,14 +190,16 @@ access, node visibility, drain controls, migration serialization, and scheduler fencing. It does not by itself provide: - a highly available PostgreSQL, Redis, or object-store deployment; -- automatic PostgreSQL backup, point-in-time recovery, or restore verification; +- automatic PostgreSQL/object backup creation or point-in-time recovery; - autoscaling policy; - central logs, metrics, traces, or alert routing; - certificate portability between independently managed ingress providers; - automatic reconciliation of every possible module side effect; - a service-level availability guarantee. -Those are deployment and module-adoption requirements. Before claiming high +The deployer verifies and gates migrations on signed coordinated backup and +isolated-restore evidence, but backup capture and restoration remain owned by +the selected state-service providers. Before claiming high availability, drill replica loss, rolling replacement, session continuity, job redelivery, scheduler failover, migration exclusion, object-store outage, and a coordinated database/object/key restore. Recovery rules and evidence are diff --git a/docs/backup-evidence-keyring.schema.json b/docs/backup-evidence-keyring.schema.json new file mode 100644 index 0000000..4215fad --- /dev/null +++ b/docs/backup-evidence-keyring.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://govoplan.add-ideas.de/schemas/backup-evidence-keyring-v1.json", + "title": "GovOPlaN backup evidence trust keyring", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "purpose", "keys"], + "properties": { + "schema_version": { "const": "1" }, + "purpose": { "const": "govoplan-backup-evidence" }, + "keys": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "key_id", + "algorithm", + "status", + "public_key_pem", + "not_before", + "expires_at" + ], + "properties": { + "key_id": { "type": "string", "minLength": 1, "maxLength": 128 }, + "algorithm": { "const": "ed25519" }, + "status": { "enum": ["active", "retired", "revoked"] }, + "public_key_pem": { "type": "string", "minLength": 1, "maxLength": 8192 }, + "not_before": { "type": "string", "format": "date-time" }, + "expires_at": { "type": "string", "format": "date-time" } + } + } + } + } +} diff --git a/docs/backup-evidence.schema.json b/docs/backup-evidence.schema.json new file mode 100644 index 0000000..fe371bb --- /dev/null +++ b/docs/backup-evidence.schema.json @@ -0,0 +1,255 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://govoplan.add-ideas.de/schemas/backup-evidence-v1.json", + "title": "GovOPlaN coordinated backup and restore evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "evidence_id", + "installation_id", + "deployment_subject", + "release", + "recovery_point", + "components", + "restore_drill", + "issued_at", + "expires_at", + "revoked", + "signatures" + ], + "properties": { + "schema_version": { "const": "1" }, + "evidence_id": { "$ref": "#/$defs/token" }, + "installation_id": { "$ref": "#/$defs/token" }, + "deployment_subject": { + "type": "object", + "additionalProperties": false, + "required": ["profile", "topology", "subject_ref"], + "properties": { + "profile": { "enum": ["evaluation", "self-hosted"] }, + "topology": { "$ref": "#/$defs/token" }, + "subject_ref": { "$ref": "#/$defs/reference" } + } + }, + "release": { + "type": "object", + "additionalProperties": false, + "required": [ + "channel", + "version", + "manifest_sha256", + "composition_sha256", + "api_image", + "web_image" + ], + "properties": { + "channel": { "$ref": "#/$defs/token" }, + "version": { "$ref": "#/$defs/token" }, + "manifest_sha256": { "$ref": "#/$defs/sha256" }, + "composition_sha256": { "$ref": "#/$defs/sha256" }, + "api_image": { "$ref": "#/$defs/digest_image" }, + "web_image": { "$ref": "#/$defs/digest_image" } + } + }, + "recovery_point": { + "type": "object", + "additionalProperties": false, + "required": ["id", "captured_at", "consistency", "rpo_seconds", "write_fence"], + "properties": { + "id": { "$ref": "#/$defs/token" }, + "captured_at": { "type": "string", "format": "date-time" }, + "consistency": { + "enum": ["provider-atomic", "application-quiesced", "transaction-consistent"] + }, + "rpo_seconds": { "$ref": "#/$defs/duration" }, + "write_fence": { + "type": "object", + "additionalProperties": false, + "required": ["mode", "token_sha256", "established_at"], + "properties": { + "mode": { + "enum": ["provider-snapshot", "application-quiesce", "transaction-boundary"] + }, + "token_sha256": { "$ref": "#/$defs/sha256" }, + "established_at": { "type": "string", "format": "date-time" } + } + } + } + }, + "components": { + "type": "object", + "additionalProperties": false, + "required": ["database", "objects", "configuration", "key_custody"], + "properties": { + "database": { "$ref": "#/$defs/database" }, + "objects": { "$ref": "#/$defs/objects" }, + "configuration": { "$ref": "#/$defs/configuration" }, + "key_custody": { "$ref": "#/$defs/key_custody" } + } + }, + "restore_drill": { + "type": "object", + "additionalProperties": false, + "required": [ + "drill_id", + "recovery_point_id", + "started_at", + "completed_at", + "isolated_target_ref", + "release_manifest_sha256", + "migration_heads_sha256", + "representative_object_manifest_sha256", + "database_verified", + "objects_verified", + "configuration_verified", + "key_custody_verified", + "semantic_checks", + "measured_rpo_seconds", + "measured_rto_seconds", + "evidence_ref" + ], + "properties": { + "drill_id": { "$ref": "#/$defs/token" }, + "recovery_point_id": { "$ref": "#/$defs/token" }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" }, + "isolated_target_ref": { "$ref": "#/$defs/reference" }, + "release_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "migration_heads_sha256": { "$ref": "#/$defs/sha256" }, + "representative_object_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "database_verified": { "const": true }, + "objects_verified": { "const": true }, + "configuration_verified": { "const": true }, + "key_custody_verified": { "const": true }, + "semantic_checks": { + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "status", "evidence_ref"], + "properties": { + "id": { "$ref": "#/$defs/token" }, + "status": { "const": "passed" }, + "evidence_ref": { "$ref": "#/$defs/reference" } + } + } + }, + "measured_rpo_seconds": { "$ref": "#/$defs/duration" }, + "measured_rto_seconds": { "$ref": "#/$defs/duration" }, + "evidence_ref": { "$ref": "#/$defs/reference" } + } + }, + "issued_at": { "type": "string", "format": "date-time" }, + "expires_at": { "type": "string", "format": "date-time" }, + "revoked": { "const": false }, + "signatures": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "items": { "$ref": "#/$defs/signature" } + } + }, + "$defs": { + "token": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" + }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "digest_image": { + "type": "string", + "maxLength": 300, + "pattern": "^[^@\\s]+@sha256:[0-9a-f]{64}$" + }, + "reference": { + "type": "string", + "minLength": 3, + "maxLength": 2048, + "pattern": "^[A-Za-z][A-Za-z0-9+.-]*:[^\\s]+$" + }, + "duration": { "type": "integer", "minimum": 0, "maximum": 2592000 }, + "protected_key": { + "type": "object", + "properties": { + "protected": { "const": true }, + "encryption_key_ref": { "$ref": "#/$defs/reference" }, + "captured_at": { "type": "string", "format": "date-time" } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", "artifact_ref", "artifact_sha256", "snapshot_id", "lsn", + "protected", "encryption_key_ref", "captured_at" + ], + "properties": { + "provider": { "$ref": "#/$defs/token" }, + "artifact_ref": { "$ref": "#/$defs/reference" }, + "artifact_sha256": { "$ref": "#/$defs/sha256" }, + "snapshot_id": { "$ref": "#/$defs/token" }, + "lsn": { "type": "string", "minLength": 1, "maxLength": 256 }, + "protected": { "const": true }, + "encryption_key_ref": { "$ref": "#/$defs/reference" }, + "captured_at": { "type": "string", "format": "date-time" } + } + }, + "objects": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", "artifact_ref", "manifest_sha256", "version_id", + "object_count", "total_bytes", "protected", "encryption_key_ref", "captured_at" + ], + "properties": { + "provider": { "$ref": "#/$defs/token" }, + "artifact_ref": { "$ref": "#/$defs/reference" }, + "manifest_sha256": { "$ref": "#/$defs/sha256" }, + "version_id": { "$ref": "#/$defs/token" }, + "object_count": { "type": "integer", "minimum": 0 }, + "total_bytes": { "type": "integer", "minimum": 0 }, + "protected": { "const": true }, + "encryption_key_ref": { "$ref": "#/$defs/reference" }, + "captured_at": { "type": "string", "format": "date-time" } + } + }, + "configuration": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_ref", "sha256", "protected", "encryption_key_ref", "captured_at"], + "properties": { + "artifact_ref": { "$ref": "#/$defs/reference" }, + "sha256": { "$ref": "#/$defs/sha256" }, + "protected": { "const": true }, + "encryption_key_ref": { "$ref": "#/$defs/reference" }, + "captured_at": { "type": "string", "format": "date-time" } + } + }, + "key_custody": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"], + "properties": { + "provider": { "$ref": "#/$defs/token" }, + "keyset_ref": { "$ref": "#/$defs/reference" }, + "keyset_version": { "$ref": "#/$defs/token" }, + "recoverable": { "const": true }, + "captured_at": { "type": "string", "format": "date-time" } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": ["key_id", "algorithm", "value"], + "properties": { + "key_id": { "$ref": "#/$defs/token" }, + "algorithm": { "const": "ed25519" }, + "value": { "type": "string", "minLength": 1, "maxLength": 256 } + } + } + } +} diff --git a/tests/test_backup_evidence.py b/tests/test_backup_evidence.py new file mode 100644 index 0000000..bda995d --- /dev/null +++ b/tests/test_backup_evidence.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import base64 +from contextlib import redirect_stderr, redirect_stdout +from datetime import UTC, datetime, timedelta +import hashlib +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + +META_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(META_ROOT / "tools" / "deployment")) + +from govoplan_deploy.backup_evidence import verify_backup_evidence # noqa: E402 +from govoplan_deploy.bundle import ( # noqa: E402 + atomic_write, + bundle_paths, + canonical_json, + read_env, +) +from govoplan_deploy.cli import main as deploy_main # noqa: E402 +from govoplan_deploy.distribution import ( # noqa: E402 + DistributionError, + canonical_signed_payload, + canonical_json as canonical_distribution_json, +) +from govoplan_deploy.model import default_spec, parse_spec # noqa: E402 +from govoplan_deploy.planning import ( # noqa: E402 + release_change_requires_backup, + verify_stored_backup_evidence, +) + + +class BackupEvidenceTests(unittest.TestCase): + def setUp(self) -> None: + self.now = datetime(2026, 8, 3, 12, tzinfo=UTC) + self.private = Ed25519PrivateKey.generate() + public = ( + self.private.public_key() + .public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("ascii") + ) + self.keyring = { + "schema_version": "1", + "purpose": "govoplan-backup-evidence", + "keys": [ + { + "key_id": "backup-controller-1", + "algorithm": "ed25519", + "status": "active", + "public_key_pem": public, + "not_before": (self.now - timedelta(days=1)).isoformat(), + "expires_at": (self.now + timedelta(days=365)).isoformat(), + } + ], + } + self.release = { + "channel": "stable", + "version": "1.2.3", + "manifest_sha256": "a" * 64, + "composition_sha256": "b" * 64, + "api_image": "registry.example/api@sha256:" + "c" * 64, + "web_image": "registry.example/web@sha256:" + "d" * 64, + } + + def test_verifies_coordinated_restore_drill_and_release_binding(self) -> None: + summary = verify_backup_evidence( + self._evidence(), + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + self.assertEqual("recovery-1", summary["recovery_point_id"]) + self.assertEqual("restore-1", summary["restore_drill_id"]) + self.assertEqual("backup-controller-1", summary["signature_key_id"]) + + def test_tampering_staleness_and_partial_restore_fail_closed(self) -> None: + tampered = self._evidence() + tampered["components"]["objects"]["object_count"] = 999 + with self.assertRaisesRegex(DistributionError, "signature verification"): + verify_backup_evidence( + tampered, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + stale = self._evidence(captured=self.now - timedelta(days=2)) + with self.assertRaisesRegex(DistributionError, "stale"): + verify_backup_evidence( + stale, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + partial = self._evidence() + partial["restore_drill"]["objects_verified"] = False + partial["signatures"] = [self._signature(partial)] + with self.assertRaisesRegex(DistributionError, "objects_verified"): + verify_backup_evidence( + partial, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + def test_wrong_release_key_purpose_and_component_skew_fail_closed(self) -> None: + wrong_release = dict(self.release) + wrong_release["version"] = "1.2.4" + with self.assertRaisesRegex(DistributionError, "release field"): + verify_backup_evidence( + self._evidence(), + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=wrong_release, + now=self.now, + ) + + wrong_keyring = dict(self.keyring) + wrong_keyring["purpose"] = "govoplan-runtime-distribution" + with self.assertRaisesRegex(DistributionError, "wrong purpose"): + verify_backup_evidence( + self._evidence(), + wrong_keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + skewed = self._evidence() + skewed["components"]["database"]["captured_at"] = ( + self.now - timedelta(hours=1) + ).isoformat() + skewed["signatures"] = [self._signature(skewed)] + with self.assertRaisesRegex(DistributionError, "one recovery point"): + verify_backup_evidence( + skewed, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + false_rto = self._evidence() + false_rto["restore_drill"]["measured_rto_seconds"] = 1 + false_rto["signatures"] = [self._signature(false_rto)] + with self.assertRaisesRegex(DistributionError, "RTO"): + verify_backup_evidence( + false_rto, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + now=self.now, + ) + + def test_provider_signing_tool_emits_canonical_verified_evidence(self) -> None: + self.now = datetime.now(UTC) + self.keyring["keys"][0]["not_before"] = ( + self.now - timedelta(days=1) + ).isoformat() + self.keyring["keys"][0]["expires_at"] = ( + self.now + timedelta(days=365) + ).isoformat() + evidence = self._evidence() + evidence["signatures"] = [] + + with tempfile.TemporaryDirectory(prefix="govoplan-backup-signer-") as value: + root = Path(value) + source = root / "unsigned.json" + output = root / "signed.json" + keyring = root / "keyring.json" + private_key = root / "private.pem" + atomic_write(source, canonical_json(evidence), mode=0o600) + atomic_write(keyring, canonical_json(self.keyring), mode=0o600) + atomic_write( + private_key, + self.private.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ), + mode=0o600, + ) + + result = subprocess.run( + [ + sys.executable, + str(META_ROOT / "tools/deployment/sign-backup-evidence.py"), + "--input", + str(source), + "--output", + str(output), + "--trusted-keyring", + str(keyring), + "--signing-key", + f"backup-controller-1={private_key}", + ], + cwd=META_ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(0, result.returncode, result.stderr) + encoded = output.read_bytes() + signed = json.loads(encoded) + self.assertEqual(canonical_distribution_json(signed), encoded) + summary = verify_backup_evidence( + signed, + self.keyring, + installation_id="govoplan-test", + profile="self-hosted", + release=self.release, + ) + self.assertEqual("backup-controller-1", summary["signature_key_id"]) + + def test_cli_adoption_gates_the_next_release_against_previous_receipt(self) -> None: + self.now = datetime.now(UTC) + self.keyring["keys"][0]["not_before"] = ( + self.now - timedelta(days=1) + ).isoformat() + self.keyring["keys"][0]["expires_at"] = ( + self.now + timedelta(days=365) + ).isoformat() + evidence = self._evidence() + encoded_evidence = canonical_distribution_json(evidence) + encoded_keyring = canonical_distribution_json(self.keyring) + + with tempfile.TemporaryDirectory(prefix="govoplan-backup-evidence-") as value: + paths = bundle_paths(Path(value)) + paths.root.chmod(0o700) + raw = default_spec( + installation_id="govoplan-test", + profile="self-hosted", + public_url="https://govoplan.example.test", + ingress_mode="existing-proxy", + trusted_proxy_cidrs=("127.0.0.1/32",), + ).to_dict() + raw["release"] = { + **raw["release"], + **self.release, + } + current = parse_spec(raw) + atomic_write(paths.spec, canonical_json(current.to_dict()), mode=0o600) + source_evidence = paths.root / "source-backup.json" + source_keyring = paths.root / "source-keyring.json" + atomic_write(source_evidence, encoded_evidence, mode=0o600) + atomic_write(source_keyring, encoded_keyring, mode=0o600) + + output = io.StringIO() + with redirect_stdout(output), redirect_stderr(output): + result = deploy_main( + [ + "verify-backup", + "--directory", + str(paths.root), + "--evidence", + str(source_evidence), + "--evidence-sha256", + hashlib.sha256(encoded_evidence).hexdigest(), + "--trusted-keyring", + str(source_keyring), + "--adopt", + ] + ) + self.assertEqual(0, result, output.getvalue()) + runtime_environment = read_env(paths.env) + self.assertEqual( + "verified", + runtime_environment["GOVOPLAN_BACKUP_EVIDENCE_STATE"], + ) + self.assertEqual( + "recovery-1", + runtime_environment["GOVOPLAN_BACKUP_RECOVERY_POINT_ID"], + ) + self.assertNotIn("snapshot:postgres", str(runtime_environment)) + self.assertNotIn("urn:kms", str(runtime_environment)) + + receipt = { + "installation_id": current.installation_id, + "profile": current.profile, + "release": dict(self.release), + } + atomic_write(paths.receipt, canonical_json(receipt), mode=0o600) + target_raw = current.to_dict() + target_raw["release"]["version"] = "1.2.4" + target_raw["release"]["manifest_sha256"] = "9" * 64 + target = parse_spec(target_raw) + + self.assertTrue(release_change_requires_backup(target, receipt)) + summary = verify_stored_backup_evidence( + target, + paths, + receipt=receipt, + ) + self.assertEqual("recovery-1", summary["recovery_point_id"]) + + def _evidence(self, *, captured: datetime | None = None) -> dict[str, object]: + captured = captured or self.now - timedelta(hours=2) + started = captured + timedelta(minutes=15) + completed = captured + timedelta(minutes=30) + issued = completed + timedelta(minutes=10) + artifact_time = captured.isoformat() + payload: dict[str, object] = { + "schema_version": "1", + "evidence_id": "backup-1", + "installation_id": "govoplan-test", + "deployment_subject": { + "profile": "self-hosted", + "topology": "compose", + "subject_ref": "urn:govoplan:installation:govoplan-test", + }, + "release": dict(self.release), + "recovery_point": { + "id": "recovery-1", + "captured_at": captured.isoformat(), + "consistency": "application-quiesced", + "rpo_seconds": 300, + "write_fence": { + "mode": "application-quiesce", + "token_sha256": "e" * 64, + "established_at": captured.isoformat(), + }, + }, + "components": { + "database": { + "provider": "postgres", + "artifact_ref": "snapshot:postgres:backup-1", + "artifact_sha256": "1" * 64, + "snapshot_id": "pg-snapshot-1", + "lsn": "0/16B6C50", + "protected": True, + "encryption_key_ref": "urn:kms:key:database-backup", + "captured_at": artifact_time, + }, + "objects": { + "provider": "s3", + "artifact_ref": "s3://backup/govoplan-test/recovery-1", + "manifest_sha256": "2" * 64, + "version_id": "object-snapshot-1", + "object_count": 4, + "total_bytes": 1024, + "protected": True, + "encryption_key_ref": "urn:kms:key:object-backup", + "captured_at": artifact_time, + }, + "configuration": { + "artifact_ref": "backup:configuration:recovery-1", + "sha256": "3" * 64, + "protected": True, + "encryption_key_ref": "urn:kms:key:configuration-backup", + "captured_at": artifact_time, + }, + "key_custody": { + "provider": "kms", + "keyset_ref": "urn:kms:keyset:govoplan-test", + "keyset_version": "version-4", + "recoverable": True, + "captured_at": artifact_time, + }, + }, + "restore_drill": { + "drill_id": "restore-1", + "recovery_point_id": "recovery-1", + "started_at": started.isoformat(), + "completed_at": completed.isoformat(), + "isolated_target_ref": "urn:govoplan:restore-target:restore-1", + "release_manifest_sha256": self.release["manifest_sha256"], + "migration_heads_sha256": "4" * 64, + "representative_object_manifest_sha256": "2" * 64, + "database_verified": True, + "objects_verified": True, + "configuration_verified": True, + "key_custody_verified": True, + "semantic_checks": [ + { + "id": "institutional-journey", + "status": "passed", + "evidence_ref": "evidence:journey:institutional-1", + } + ], + "measured_rpo_seconds": 120, + "measured_rto_seconds": 900, + "evidence_ref": "evidence:restore:restore-1", + }, + "issued_at": issued.isoformat(), + "expires_at": (self.now + timedelta(days=7)).isoformat(), + "revoked": False, + "signatures": [], + } + payload["signatures"] = [self._signature(payload)] + return payload + + def _signature(self, payload: dict[str, object]) -> dict[str, str]: + return { + "key_id": "backup-controller-1", + "algorithm": "ed25519", + "value": base64.b64encode( + self.private.sign(canonical_signed_payload(payload)) + ).decode("ascii"), + } + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_deployment_installer.py b/tests/test_deployment_installer.py index a0e1d90..6bfcedb 100644 --- a/tests/test_deployment_installer.py +++ b/tests/test_deployment_installer.py @@ -318,7 +318,16 @@ class DeploymentInstallerTests(unittest.TestCase): }, ) - manifest = render_kubernetes(spec, environment) + manifest = render_kubernetes( + spec, + environment, + backup_required=True, + backup_evidence={ + "evidence_sha256": "c" * 64, + "recovery_point_id": "recovery-1", + "restore_drill_id": "drill-1", + }, + ) rendered = json.dumps(manifest, sort_keys=True) kinds = [item["kind"] for item in manifest["items"]] deployments = { @@ -351,6 +360,18 @@ class DeploymentInstallerTests(unittest.TestCase): "forward-recovery", migration["metadata"]["annotations"]["govoplan.add-ideas.de/recovery-mode"], ) + self.assertEqual( + "c" * 64, + migration["metadata"]["annotations"][ + "govoplan.add-ideas.de/backup-evidence-sha256" + ], + ) + self.assertEqual( + "recovery-1", + migration["metadata"]["annotations"][ + "govoplan.add-ideas.de/recovery-point" + ], + ) config = next(item for item in manifest["items"] if item["kind"] == "ConfigMap") self.assertEqual("shared", config["data"]["GOVOPLAN_STATE_PROFILE"]) self.assertEqual("3", config["data"]["GOVOPLAN_EXPECTED_API_REPLICAS"]) diff --git a/tools/deployment/govoplan_deploy/backup_evidence.py b/tools/deployment/govoplan_deploy/backup_evidence.py new file mode 100644 index 0000000..e9b1b5b --- /dev/null +++ b/tools/deployment/govoplan_deploy/backup_evidence.py @@ -0,0 +1,500 @@ +"""Signed, provider-neutral backup and isolated-restore evidence.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +import re +from typing import Any, Mapping +from urllib.parse import urlsplit + +from .distribution import ( + DistributionError, + load_bounded_json, + verify_signed_document, +) + + +MAX_BACKUP_EVIDENCE_BYTES = 1024 * 1024 +MAX_BACKUP_KEYRING_BYTES = 1024 * 1024 +DEFAULT_MAX_BACKUP_AGE_SECONDS = 24 * 60 * 60 +MAX_COORDINATION_SKEW_SECONDS = 5 * 60 +SHA256 = re.compile(r"^[0-9a-f]{64}$") +TOKEN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") +REFERENCE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:[^\s]{1,2040}$") + + +def load_backup_evidence(path: Path) -> dict[str, Any]: + return load_bounded_json(path, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES) + + +def load_backup_keyring(path: Path) -> dict[str, Any]: + return load_bounded_json(path, maximum_bytes=MAX_BACKUP_KEYRING_BYTES) + + +def verify_backup_evidence( + payload: Mapping[str, Any], + keyring: Mapping[str, Any], + *, + installation_id: str, + profile: str, + release: Mapping[str, object], + now: datetime | None = None, + max_age_seconds: int = DEFAULT_MAX_BACKUP_AGE_SECONDS, + openssl: str = "openssl", +) -> dict[str, object]: + current = (now or datetime.now(UTC)).astimezone(UTC) + values = _validate_payload(payload, now=current, max_age_seconds=max_age_seconds) + if payload.get("installation_id") != installation_id: + raise DistributionError("backup evidence belongs to another installation") + subject = _object(payload.get("deployment_subject"), "deployment_subject") + if subject.get("profile") != profile: + raise DistributionError("backup evidence belongs to another deployment profile") + evidence_release = _object(payload.get("release"), "release") + for field in ( + "channel", + "version", + "manifest_sha256", + "composition_sha256", + "api_image", + "web_image", + ): + if evidence_release.get(field) != release.get(field): + raise DistributionError( + f"backup evidence does not match release field {field!r}" + ) + key_id = verify_signed_document( + payload, + keyring, + purpose="govoplan-backup-evidence", + label="backup evidence", + now=current, + openssl=openssl, + ) + recovery_point = _object(payload.get("recovery_point"), "recovery_point") + restore = _object(payload.get("restore_drill"), "restore_drill") + return { + "evidence_id": payload["evidence_id"], + "recovery_point_id": recovery_point["id"], + "captured_at": recovery_point["captured_at"], + "expires_at": payload["expires_at"], + "restore_drill_id": restore["drill_id"], + "restore_started_at": restore["started_at"], + "restore_completed_at": restore["completed_at"], + "measured_rpo_seconds": restore["measured_rpo_seconds"], + "measured_rto_seconds": restore["measured_rto_seconds"], + "signature_key_id": key_id, + "component_count": values["component_count"], + } + + +def _validate_payload( + payload: Mapping[str, Any], + *, + now: datetime, + max_age_seconds: int, +) -> dict[str, int]: + if max_age_seconds < 60 or max_age_seconds > 30 * 24 * 60 * 60: + raise DistributionError("backup maximum age is out of bounds") + _exact_keys( + payload, + { + "schema_version", + "evidence_id", + "installation_id", + "deployment_subject", + "release", + "recovery_point", + "components", + "restore_drill", + "issued_at", + "expires_at", + "revoked", + "signatures", + }, + "backup evidence", + ) + if payload.get("schema_version") != "1": + raise DistributionError("unsupported backup evidence schema_version") + _token(payload.get("evidence_id"), "evidence_id") + _token(payload.get("installation_id"), "installation_id") + issued = _timestamp(payload.get("issued_at"), "issued_at") + expires = _timestamp(payload.get("expires_at"), "expires_at") + if issued > now or expires <= issued or expires <= now: + raise DistributionError("backup evidence is not currently valid") + if payload.get("revoked") is not False: + raise DistributionError("backup evidence is revoked") + + subject = _object(payload.get("deployment_subject"), "deployment_subject") + _exact_keys(subject, {"profile", "topology", "subject_ref"}, "deployment_subject") + if subject.get("profile") not in {"evaluation", "self-hosted"}: + raise DistributionError("deployment_subject.profile is invalid") + _token(subject.get("topology"), "deployment_subject.topology") + _reference(subject.get("subject_ref"), "deployment_subject.subject_ref") + + release = _object(payload.get("release"), "release") + _exact_keys( + release, + { + "channel", + "version", + "manifest_sha256", + "composition_sha256", + "api_image", + "web_image", + }, + "release", + ) + _token(release.get("channel"), "release.channel") + _token(release.get("version"), "release.version") + _sha256(release.get("manifest_sha256"), "release.manifest_sha256") + _sha256(release.get("composition_sha256"), "release.composition_sha256") + _image(release.get("api_image"), "release.api_image") + _image(release.get("web_image"), "release.web_image") + + recovery = _object(payload.get("recovery_point"), "recovery_point") + _exact_keys( + recovery, + {"id", "captured_at", "consistency", "rpo_seconds", "write_fence"}, + "recovery_point", + ) + recovery_id = _token(recovery.get("id"), "recovery_point.id") + captured = _timestamp(recovery.get("captured_at"), "recovery_point.captured_at") + age = (now - captured).total_seconds() + if age < 0 or age > max_age_seconds: + raise DistributionError("backup recovery point is stale or in the future") + if recovery.get("consistency") not in { + "provider-atomic", + "application-quiesced", + "transaction-consistent", + }: + raise DistributionError("recovery_point.consistency is invalid") + declared_rpo = _bounded_integer( + recovery.get("rpo_seconds"), + "recovery_point.rpo_seconds", + maximum=30 * 24 * 60 * 60, + ) + fence = _object(recovery.get("write_fence"), "recovery_point.write_fence") + _exact_keys( + fence, + {"mode", "token_sha256", "established_at"}, + "recovery_point.write_fence", + ) + if fence.get("mode") not in { + "provider-snapshot", + "application-quiesce", + "transaction-boundary", + }: + raise DistributionError("recovery_point.write_fence.mode is invalid") + _sha256(fence.get("token_sha256"), "recovery_point.write_fence.token_sha256") + established = _timestamp( + fence.get("established_at"), + "recovery_point.write_fence.established_at", + ) + if abs((captured - established).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS: + raise DistributionError( + "backup write fence is not coordinated with recovery point" + ) + + components = _object(payload.get("components"), "components") + _exact_keys( + components, + {"database", "objects", "configuration", "key_custody"}, + "components", + ) + captured_components = [ + _database_component(components.get("database")), + _objects_component(components.get("objects")), + _configuration_component(components.get("configuration")), + _key_custody_component(components.get("key_custody")), + ] + if any( + abs((component_time - captured).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS + for component_time in captured_components + ): + raise DistributionError("backup components do not share one recovery point") + + restore = _object(payload.get("restore_drill"), "restore_drill") + _exact_keys( + restore, + { + "drill_id", + "recovery_point_id", + "started_at", + "completed_at", + "isolated_target_ref", + "release_manifest_sha256", + "migration_heads_sha256", + "representative_object_manifest_sha256", + "database_verified", + "objects_verified", + "configuration_verified", + "key_custody_verified", + "semantic_checks", + "measured_rpo_seconds", + "measured_rto_seconds", + "evidence_ref", + }, + "restore_drill", + ) + _token(restore.get("drill_id"), "restore_drill.drill_id") + if restore.get("recovery_point_id") != recovery_id: + raise DistributionError("restore drill used another recovery point") + started = _timestamp(restore.get("started_at"), "restore_drill.started_at") + completed = _timestamp(restore.get("completed_at"), "restore_drill.completed_at") + if started < captured or completed < started or completed > issued: + raise DistributionError( + "restore drill completion is outside evidence chronology" + ) + _reference(restore.get("isolated_target_ref"), "restore_drill.isolated_target_ref") + _reference(restore.get("evidence_ref"), "restore_drill.evidence_ref") + for field in ( + "release_manifest_sha256", + "migration_heads_sha256", + "representative_object_manifest_sha256", + ): + _sha256(restore.get(field), f"restore_drill.{field}") + if restore.get("release_manifest_sha256") != release.get("manifest_sha256"): + raise DistributionError("restore drill used another immutable release") + for field in ( + "database_verified", + "objects_verified", + "configuration_verified", + "key_custody_verified", + ): + if restore.get(field) is not True: + raise DistributionError(f"restore_drill.{field} must be true") + semantic = restore.get("semantic_checks") + if not isinstance(semantic, list) or not semantic or len(semantic) > 128: + raise DistributionError("restore_drill.semantic_checks must not be empty") + seen_checks: set[str] = set() + for index, raw in enumerate(semantic): + check = _object(raw, f"restore_drill.semantic_checks[{index}]") + _exact_keys( + check, + {"id", "status", "evidence_ref"}, + f"restore_drill.semantic_checks[{index}]", + ) + check_id = _token(check.get("id"), f"semantic_checks[{index}].id") + if check_id in seen_checks or check.get("status") != "passed": + raise DistributionError("restore drill semantic checks are invalid") + seen_checks.add(check_id) + _reference(check.get("evidence_ref"), f"semantic_checks[{index}].evidence_ref") + measured_rpo = _bounded_integer( + restore.get("measured_rpo_seconds"), + "restore_drill.measured_rpo_seconds", + maximum=30 * 24 * 60 * 60, + ) + measured_rto = _bounded_integer( + restore.get("measured_rto_seconds"), + "restore_drill.measured_rto_seconds", + maximum=30 * 24 * 60 * 60, + ) + if abs((completed - started).total_seconds() - measured_rto) > 5: + raise DistributionError("restore drill RTO does not match its timestamps") + if measured_rpo > declared_rpo: + raise DistributionError( + "restore drill exceeds the declared recovery point objective" + ) + _validate_signatures(payload.get("signatures")) + return {"component_count": len(captured_components)} + + +def _database_component(raw: object) -> datetime: + value = _object(raw, "components.database") + _exact_keys( + value, + { + "provider", + "artifact_ref", + "artifact_sha256", + "snapshot_id", + "lsn", + "protected", + "encryption_key_ref", + "captured_at", + }, + "components.database", + ) + _common_artifact(value, "components.database") + _token(value.get("snapshot_id"), "components.database.snapshot_id") + _bounded_text(value.get("lsn"), "components.database.lsn", maximum=256) + return _timestamp(value.get("captured_at"), "components.database.captured_at") + + +def _objects_component(raw: object) -> datetime: + value = _object(raw, "components.objects") + _exact_keys( + value, + { + "provider", + "artifact_ref", + "manifest_sha256", + "version_id", + "object_count", + "total_bytes", + "protected", + "encryption_key_ref", + "captured_at", + }, + "components.objects", + ) + _token(value.get("provider"), "components.objects.provider") + _reference(value.get("artifact_ref"), "components.objects.artifact_ref") + _sha256(value.get("manifest_sha256"), "components.objects.manifest_sha256") + _token(value.get("version_id"), "components.objects.version_id") + _bounded_integer(value.get("object_count"), "components.objects.object_count") + _bounded_integer(value.get("total_bytes"), "components.objects.total_bytes") + _protected_key_reference(value, "components.objects") + return _timestamp(value.get("captured_at"), "components.objects.captured_at") + + +def _configuration_component(raw: object) -> datetime: + value = _object(raw, "components.configuration") + _exact_keys( + value, + { + "artifact_ref", + "sha256", + "protected", + "encryption_key_ref", + "captured_at", + }, + "components.configuration", + ) + _reference(value.get("artifact_ref"), "components.configuration.artifact_ref") + _sha256(value.get("sha256"), "components.configuration.sha256") + _protected_key_reference(value, "components.configuration") + return _timestamp(value.get("captured_at"), "components.configuration.captured_at") + + +def _key_custody_component(raw: object) -> datetime: + value = _object(raw, "components.key_custody") + _exact_keys( + value, + {"provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"}, + "components.key_custody", + ) + _token(value.get("provider"), "components.key_custody.provider") + _reference(value.get("keyset_ref"), "components.key_custody.keyset_ref") + _token(value.get("keyset_version"), "components.key_custody.keyset_version") + if value.get("recoverable") is not True: + raise DistributionError("components.key_custody.recoverable must be true") + return _timestamp(value.get("captured_at"), "components.key_custody.captured_at") + + +def _common_artifact(value: Mapping[str, Any], label: str) -> None: + _token(value.get("provider"), f"{label}.provider") + _reference(value.get("artifact_ref"), f"{label}.artifact_ref") + _sha256(value.get("artifact_sha256"), f"{label}.artifact_sha256") + _protected_key_reference(value, label) + + +def _protected_key_reference(value: Mapping[str, Any], label: str) -> None: + if value.get("protected") is not True: + raise DistributionError(f"{label}.protected must be true") + _reference(value.get("encryption_key_ref"), f"{label}.encryption_key_ref") + + +def _validate_signatures(raw: object) -> None: + if not isinstance(raw, list) or not raw or len(raw) > 16: + raise DistributionError("backup evidence signatures must not be empty") + seen: set[str] = set() + for index, item in enumerate(raw): + signature = _object(item, f"signatures[{index}]") + _exact_keys(signature, {"key_id", "algorithm", "value"}, f"signatures[{index}]") + key_id = _token(signature.get("key_id"), f"signatures[{index}].key_id") + if key_id in seen or signature.get("algorithm") != "ed25519": + raise DistributionError("backup evidence signatures are invalid") + seen.add(key_id) + encoded = signature.get("value") + if not isinstance(encoded, str) or len(encoded) > 256: + raise DistributionError("backup evidence signature value is invalid") + + +def _reference(raw: object, label: str) -> str: + value = _bounded_text(raw, label, maximum=2048) + if REFERENCE.fullmatch(value) is None or "BEGIN " in value.upper(): + raise DistributionError(f"{label} must be an opaque provider reference") + parsed = urlsplit(value) + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise DistributionError(f"{label} must not contain credentials or query data") + return value + + +def _object(raw: object, label: str) -> dict[str, Any]: + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise DistributionError(f"{label} must be an object") + return raw + + +def _exact_keys(value: Mapping[str, Any], keys: set[str], label: str) -> None: + if set(value) != keys: + missing = sorted(keys - set(value)) + extra = sorted(set(value) - keys) + detail = [] + if missing: + detail.append("missing " + ", ".join(missing)) + if extra: + detail.append("unknown " + ", ".join(extra)) + raise DistributionError(f"{label} has invalid fields: {'; '.join(detail)}") + + +def _timestamp(raw: object, label: str) -> datetime: + value = _bounded_text(raw, label, maximum=64) + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise DistributionError(f"{label} must be an RFC3339 timestamp") from exc + if parsed.tzinfo is None: + raise DistributionError(f"{label} must include a timezone") + return parsed.astimezone(UTC) + + +def _token(raw: object, label: str) -> str: + value = _bounded_text(raw, label, maximum=128) + if TOKEN.fullmatch(value) is None: + raise DistributionError(f"{label} is invalid") + return value + + +def _sha256(raw: object, label: str) -> str: + value = _bounded_text(raw, label, maximum=64) + if SHA256.fullmatch(value) is None: + raise DistributionError(f"{label} must be a lowercase SHA-256 digest") + return value + + +def _image(raw: object, label: str) -> str: + value = _bounded_text(raw, label, maximum=300) + if IMAGE.fullmatch(value) is None: + raise DistributionError(f"{label} must be an OCI image pinned by sha256") + return value + + +def _bounded_text(raw: object, label: str, *, maximum: int) -> str: + if not isinstance(raw, str) or not raw or len(raw) > maximum or "\n" in raw: + raise DistributionError(f"{label} is invalid") + return raw + + +def _bounded_integer( + raw: object, + label: str, + *, + maximum: int = 2**63 - 1, +) -> int: + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0 or raw > maximum: + raise DistributionError(f"{label} is out of bounds") + return raw + + +__all__ = [ + "DEFAULT_MAX_BACKUP_AGE_SECONDS", + "MAX_BACKUP_EVIDENCE_BYTES", + "MAX_BACKUP_KEYRING_BYTES", + "load_backup_evidence", + "load_backup_keyring", + "verify_backup_evidence", +] diff --git a/tools/deployment/govoplan_deploy/bundle.py b/tools/deployment/govoplan_deploy/bundle.py index d2100ea..f106373 100644 --- a/tools/deployment/govoplan_deploy/bundle.py +++ b/tools/deployment/govoplan_deploy/bundle.py @@ -28,7 +28,26 @@ PLAN_FILENAME = "plan.json" RECEIPT_FILENAME = "receipt.json" MANIFEST_FILENAME = "distribution-manifest.json" KEYRING_FILENAME = "distribution-keyring.json" +BACKUP_EVIDENCE_FILENAME = "backup-evidence.json" +BACKUP_KEYRING_FILENAME = "backup-keyring.json" +BACKUP_VERIFICATION_FILENAME = "backup-verification.json" LOCK_FILENAME = ".deployment.lock" +BACKUP_RUNTIME_ENV_KEYS = ( + "GOVOPLAN_BACKUP_EVIDENCE_STATE", + "GOVOPLAN_BACKUP_EVIDENCE_ID", + "GOVOPLAN_BACKUP_RECOVERY_POINT_ID", + "GOVOPLAN_BACKUP_RESTORE_DRILL_ID", + "GOVOPLAN_BACKUP_EVIDENCE_SHA256", + "GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256", + "GOVOPLAN_BACKUP_CAPTURED_AT", + "GOVOPLAN_BACKUP_EXPIRES_AT", + "GOVOPLAN_BACKUP_RESTORE_STARTED_AT", + "GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT", + "GOVOPLAN_BACKUP_VERIFIED_AT", + "GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS", + "GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS", + "GOVOPLAN_BACKUP_COMPONENT_COUNT", +) RUNTIME_ENV_KEYS = ( "APP_ENV", "GOVOPLAN_INSTALL_PROFILE", @@ -78,6 +97,7 @@ RUNTIME_ENV_KEYS = ( "FILE_STORAGE_S3_BUCKET", "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + *BACKUP_RUNTIME_ENV_KEYS, ) @@ -95,6 +115,9 @@ class BundlePaths: receipt: Path manifest: Path keyring: Path + backup_evidence: Path + backup_keyring: Path + backup_verification: Path lock: Path @@ -116,6 +139,9 @@ def bundle_paths(root: Path) -> BundlePaths: receipt=resolved / RECEIPT_FILENAME, manifest=resolved / MANIFEST_FILENAME, keyring=resolved / KEYRING_FILENAME, + backup_evidence=resolved / BACKUP_EVIDENCE_FILENAME, + backup_keyring=resolved / BACKUP_KEYRING_FILENAME, + backup_verification=resolved / BACKUP_VERIFICATION_FILENAME, lock=resolved / LOCK_FILENAME, ) diff --git a/tools/deployment/govoplan_deploy/cli.py b/tools/deployment/govoplan_deploy/cli.py index 1efe09b..82d60b5 100644 --- a/tools/deployment/govoplan_deploy/cli.py +++ b/tools/deployment/govoplan_deploy/cli.py @@ -20,7 +20,14 @@ from typing import Iterator, Mapping, Sequence from urllib.error import URLError from urllib.request import urlopen +from .backup_evidence import ( + DEFAULT_MAX_BACKUP_AGE_SECONDS, + MAX_BACKUP_EVIDENCE_BYTES, + MAX_BACKUP_KEYRING_BYTES, + verify_backup_evidence, +) from .bundle import ( + BACKUP_RUNTIME_ENV_KEYS, atomic_write, bundle_paths, canonical_json, @@ -72,7 +79,12 @@ from .kubernetes import ( render_kubernetes, write_secret_creation_hint, ) -from .planning import DeploymentPlan, build_plan +from .planning import ( + DeploymentPlan, + build_plan, + release_change_requires_backup, + verify_stored_backup_evidence, +) from .recovery import ( DeploymentOperationJournal, list_operations, @@ -183,6 +195,22 @@ def build_parser() -> argparse.ArgumentParser: help="Load verified archives into Docker using fixed image-load commands.", ) + verify_backup = subparsers.add_parser( + "verify-backup", + help="Verify and optionally adopt signed coordinated backup evidence.", + ) + _directory_argument(verify_backup) + evidence_source = verify_backup.add_mutually_exclusive_group(required=True) + evidence_source.add_argument("--evidence", type=Path) + evidence_source.add_argument("--evidence-url") + verify_backup.add_argument("--evidence-sha256", required=True) + verify_backup.add_argument("--trusted-keyring", type=Path, required=True) + verify_backup.add_argument( + "--allow-private-evidence-host", + action="store_true", + ) + verify_backup.add_argument("--adopt", action="store_true") + kubernetes = subparsers.add_parser( "render-kubernetes", help="Export the stateless multi-host runtime for Kubernetes.", @@ -392,6 +420,8 @@ def main(argv: Sequence[str] | None = None) -> int: return _verify_release(args) if args.command == "verify-offline-images": return _verify_offline_images(args) + if args.command == "verify-backup": + return _verify_backup(args) if args.command == "render-kubernetes": return _render_kubernetes(args) if args.command == "verify-kubernetes": @@ -534,15 +564,17 @@ def _render_or_doctor(args: argparse.Namespace) -> int: def _apply(args: argparse.Namespace) -> int: paths = bundle_paths(args.directory) - spec = load_spec(paths.spec) - if args.allow_unverified_images and spec.profile != "evaluation": - raise ValueError( - "--allow-unverified-images is restricted to evaluation installations" - ) ensure_private_directory(paths.root) with _deployment_lock(paths.lock): + spec = load_spec(paths.spec) + previous_receipt = _read_json_object(paths.receipt) + backup_required = release_change_requires_backup(spec, previous_receipt) + if args.allow_unverified_images and spec.profile != "evaluation": + raise ValueError( + "--allow-unverified-images is restricted to evaluation installations" + ) secrets = reconcile_runtime_environment(spec, read_env(paths.env)) - _write_bundle(spec, paths, secrets) + secrets = _write_bundle(spec, paths, secrets) plan = build_plan(spec, paths, include_host_checks=True) _write_plan(paths.plan, plan) effective_errors = [ @@ -577,6 +609,36 @@ def _apply(args: argparse.Namespace) -> int: paths, plan=plan.to_dict(), ) + try: + if backup_required: + backup_summary = verify_stored_backup_evidence( + spec, + paths, + receipt=previous_receipt, + ) + journal.record( + "backup-evidence-verified", + "succeeded", + dict(backup_summary), + ) + else: + journal.record( + "backup-evidence-not-required", + "succeeded", + {"release_change": False}, + ) + except BaseException as exc: + journal.record( + "backup-evidence-rejected", + "blocked", + { + "phase": "preflight", + "exception_type": type(exc).__name__, + "migration_started": False, + }, + ) + journal.failed(exc) + raise compose = [ docker, "compose", @@ -624,6 +686,29 @@ def _apply(args: argparse.Namespace) -> int: "succeeded", {"services": mutable_runtime_services, "timeout_seconds": 120}, ) + if backup_required: + try: + backup_summary = verify_stored_backup_evidence( + spec, + paths, + receipt=previous_receipt, + ) + except BaseException as exc: + journal.record( + "backup-evidence-rejected", + "blocked", + { + "phase": "migration-boundary", + "exception_type": type(exc).__name__, + "migration_started": False, + }, + ) + raise + journal.record( + "backup-evidence-reverified", + "succeeded", + dict(backup_summary), + ) journal.migration_started() _run([*compose, "run", "--rm", "migrate"], cwd=paths.root) journal.migration_completed() @@ -898,6 +983,108 @@ def _verify_offline_images(args: argparse.Namespace) -> int: return 0 +def _verify_backup(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + ensure_private_directory(paths.root) + with _deployment_lock(paths.lock): + return _verify_backup_locked(args, paths) + + +def _verify_backup_locked(args: argparse.Namespace, paths) -> int: + spec = load_spec(paths.spec) + expected_digest = str(args.evidence_sha256 or "").strip().lower() + if len(expected_digest) != 64 or any( + character not in "0123456789abcdef" for character in expected_digest + ): + raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest") + if args.evidence_url: + encoded_evidence = fetch_bounded_https( + str(args.evidence_url), + maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES, + allow_private_host=args.allow_private_evidence_host, + ) + evidence = decode_json_bytes(encoded_evidence, label="backup evidence") + else: + evidence_path = args.evidence.expanduser().resolve() + encoded_evidence = read_bounded_bytes( + evidence_path, + maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES, + ) + evidence = decode_json_bytes(encoded_evidence, label="backup evidence") + if encoded_evidence != canonical_distribution_json(evidence): + raise DistributionError("backup evidence is not canonical JSON") + if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest: + raise DistributionError("backup evidence SHA-256 does not match") + keyring_path = args.trusted_keyring.expanduser().resolve() + keyring = load_bounded_json( + keyring_path, + maximum_bytes=MAX_BACKUP_KEYRING_BYTES, + ) + encoded_keyring = canonical_distribution_json(keyring) + receipt = _read_json_object(paths.receipt) + previous_release = receipt.get("release") if receipt else None + release: Mapping[str, object] = ( + previous_release + if isinstance(previous_release, Mapping) + else { + "channel": spec.release.channel, + "version": spec.release.version, + "manifest_sha256": spec.release.manifest_sha256, + "composition_sha256": spec.release.composition_sha256, + "api_image": spec.release.api_image, + "web_image": spec.release.web_image, + } + ) + summary = verify_backup_evidence( + evidence, + keyring, + installation_id=spec.installation_id, + profile=spec.profile, + release=release, + max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS, + ) + print( + "Verified coordinated recovery point " + f"{summary['recovery_point_id']} with restore drill " + f"{summary['restore_drill_id']} and trusted key " + f"{summary['signature_key_id']}." + ) + if not args.adopt: + return 0 + verification = { + "schema_version": 1, + "evidence_sha256": expected_digest, + "keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(), + "signature_key_id": summary["signature_key_id"], + "verified_at": _now(), + "evidence_id": summary["evidence_id"], + "recovery_point_id": summary["recovery_point_id"], + "restore_drill_id": summary["restore_drill_id"], + "release_manifest_sha256": release.get("manifest_sha256"), + "captured_at": summary["captured_at"], + "expires_at": summary["expires_at"], + "restore_started_at": summary["restore_started_at"], + "restore_completed_at": summary["restore_completed_at"], + "measured_rpo_seconds": summary["measured_rpo_seconds"], + "measured_rto_seconds": summary["measured_rto_seconds"], + "component_count": summary["component_count"], + } + atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600) + atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600) + atomic_write( + paths.backup_verification, + canonical_json(verification), + mode=0o600, + ) + _write_bundle( + spec, + paths, + reconcile_runtime_environment(spec, read_env(paths.env)), + ) + print(f"Adopted signed backup evidence in {paths.root}.") + return 0 + + def _selected_dependency_images( spec: InstallationSpec, *, @@ -929,6 +1116,30 @@ def _render_kubernetes(args: argparse.Namespace) -> int: paths = bundle_paths(args.directory) spec = load_spec(paths.spec) environment = reconcile_runtime_environment(spec, read_env(paths.env)) + environment.update(_backup_runtime_environment(spec, paths)) + receipt = _read_json_object(paths.receipt) + backup_required = release_change_requires_backup(spec, receipt) + backup_summary: Mapping[str, object] | None = None + evidence_files = ( + paths.backup_evidence, + paths.backup_keyring, + paths.backup_verification, + ) + if backup_required: + backup_summary = verify_stored_backup_evidence( + spec, + paths, + receipt=receipt, + ) + elif all(path.is_file() for path in evidence_files): + try: + backup_summary = verify_stored_backup_evidence( + spec, + paths, + receipt=receipt, + ) + except (DistributionError, OSError): + backup_summary = None manifest = render_kubernetes( spec, environment, @@ -936,6 +1147,8 @@ def _render_kubernetes(args: argparse.Namespace) -> int: secret_name=args.secret_name, tls_secret_name=args.tls_secret_name, ingress_class_name=args.ingress_class_name, + backup_required=backup_required, + backup_evidence=backup_summary, ) output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve() atomic_write(output, canonical_json(manifest), mode=0o600) @@ -1027,6 +1240,7 @@ def _deployment_receipt( return { "schema_version": 1, "installation_id": spec.installation_id, + "profile": spec.profile, "applied_at": _now(), "spec_sha256": digest_json(spec.to_dict()), "compose_sha256": digest_json(render_compose(spec)), @@ -1064,6 +1278,16 @@ def _deployment_receipt( } +def _read_json_object(path: Path) -> dict[str, object]: + if not path.is_file(): + return {} + try: + value = load_bounded_json(path, maximum_bytes=64 * 1024) + except (DistributionError, OSError): + return {} + return value + + def _updated_spec( current: InstallationSpec, args: argparse.Namespace ) -> InstallationSpec: @@ -1222,10 +1446,12 @@ def _write_bundle( spec: InstallationSpec, paths, secrets: Mapping[str, str], -) -> None: +) -> dict[str, str]: ensure_private_directory(paths.root) + runtime_environment = dict(secrets) + runtime_environment.update(_backup_runtime_environment(spec, paths)) atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600) - write_env(paths.env, secrets) + write_env(paths.env, runtime_environment) atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600) atomic_write( paths.load_balancer_config, @@ -1247,6 +1473,62 @@ def _write_bundle( render_garage_config().encode("utf-8"), mode=0o644, ) + return dict(sorted(runtime_environment.items())) + + +def _backup_runtime_environment( + spec: InstallationSpec, + paths, +) -> dict[str, str]: + values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS} + evidence_files = ( + paths.backup_evidence, + paths.backup_keyring, + paths.backup_verification, + ) + if not any(path.is_file() for path in evidence_files): + values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent" + return values + if not all(path.is_file() for path in evidence_files): + values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid" + return values + verification = _read_json_object(paths.backup_verification) + try: + summary = verify_stored_backup_evidence( + spec, + paths, + receipt=_read_json_object(paths.receipt), + ) + except (DistributionError, OSError): + values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid" + return values + values.update( + { + "GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified", + "GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]), + "GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]), + "GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]), + "GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]), + "GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str( + verification["release_manifest_sha256"] + ), + "GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]), + "GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]), + "GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]), + "GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str( + summary["restore_completed_at"] + ), + "GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]), + "GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str( + summary["measured_rpo_seconds"] + ), + "GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str( + summary["measured_rto_seconds"] + ), + "GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]), + } + ) + return values def _write_plan(path: Path, plan: DeploymentPlan) -> None: diff --git a/tools/deployment/govoplan_deploy/distribution.py b/tools/deployment/govoplan_deploy/distribution.py index 98d79ce..af6c258 100644 --- a/tools/deployment/govoplan_deploy/distribution.py +++ b/tools/deployment/govoplan_deploy/distribution.py @@ -74,7 +74,9 @@ def read_bounded_bytes(path: Path, *, maximum_bytes: int) -> bytes: try: opened = os.fstat(descriptor) if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes: - raise DistributionError(f"trusted JSON file is invalid or too large: {path}") + raise DistributionError( + f"trusted JSON file is invalid or too large: {path}" + ) chunks: list[bytes] = [] total = 0 while True: @@ -109,7 +111,9 @@ def fetch_bounded_https( if parsed.scheme != "https" or not parsed.hostname: raise DistributionError("distribution downloads require an absolute HTTPS URL") if parsed.username or parsed.password or parsed.fragment: - raise DistributionError("distribution URL must not contain credentials or a fragment") + raise DistributionError( + "distribution URL must not contain credentials or a fragment" + ) if not allow_private_host: _require_public_host(parsed.hostname) request = Request(url, headers={"Accept": "application/json"}) @@ -172,9 +176,11 @@ def validate_manifest( raise DistributionError( f"distribution channel is {channel!r}, expected {expected_channel!r}" ) - if isinstance(payload.get("sequence"), bool) or not isinstance( - payload.get("sequence"), int - ) or int(payload["sequence"]) < 1: + if ( + isinstance(payload.get("sequence"), bool) + or not isinstance(payload.get("sequence"), int) + or int(payload["sequence"]) < 1 + ): raise DistributionError("distribution sequence must be a positive integer") _token(payload.get("version"), "version", maximum=128, pattern=TOKEN) issued = _datetime(payload.get("issued_at"), "issued_at") @@ -283,11 +289,41 @@ def verify_manifest( ) -> str: current = (now or datetime.now(UTC)).astimezone(UTC) validate_manifest(payload, expected_channel=expected_channel, now=current) - keys = _trusted_keys(keyring, now=current) + return verify_signed_document( + payload, + keyring, + purpose="govoplan-runtime-distribution", + label="distribution", + now=current, + openssl=openssl, + ) + + +def verify_signed_document( + payload: Mapping[str, Any], + keyring: Mapping[str, Any], + *, + purpose: str, + label: str, + now: datetime, + openssl: str = "openssl", +) -> str: + keys = _trusted_keys(keyring, now=now, purpose=purpose, label=label) signed = canonical_signed_payload(payload) failures: list[str] = [] - for item in payload["signatures"]: + signatures = payload.get("signatures") + if not isinstance(signatures, list) or not signatures: + raise DistributionError(f"{label} has no signatures") + for index, raw in enumerate(signatures): + item = _object(raw, f"{label}.signatures[{index}]") + _exact_keys( + item, + required={"key_id", "algorithm", "value"}, + label=f"{label}.signatures[{index}]", + ) key_id = str(item["key_id"]) + if KEY_ID.fullmatch(key_id) is None or item.get("algorithm") != "ed25519": + raise DistributionError(f"{label} signature is invalid") public_key = keys.get(key_id) if public_key is None: continue @@ -303,8 +339,10 @@ def verify_manifest( failures.append(f"{key_id}: {exc}") continue return key_id - detail = "; ".join(failures) if failures else "no signature used an active trusted key" - raise DistributionError(f"distribution signature verification failed: {detail}") + detail = ( + "; ".join(failures) if failures else "no signature used an active trusted key" + ) + raise DistributionError(f"{label} signature verification failed: {detail}") def verify_manifest_binding( @@ -319,7 +357,9 @@ def verify_manifest_binding( dependencies: Mapping[str, str], ) -> None: if payload.get("channel") != channel or payload.get("version") != version: - raise DistributionError("stored manifest does not match release channel/version") + raise DistributionError( + "stored manifest does not match release channel/version" + ) images = _object(payload.get("images"), "images") if _object(images.get("api"), "images.api").get("index") != api_image: raise DistributionError("stored manifest does not match API image") @@ -372,9 +412,9 @@ def verify_offline_image_index( archive = root / archive_relative if reference in references: raise DistributionError("offline image index contains duplicate references") - if _sha256_regular_file(archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES) != _sha256( - value.get("sha256"), "offline image sha256" - ): + if _sha256_regular_file( + archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES + ) != _sha256(value.get("sha256"), "offline image sha256"): raise DistributionError(f"offline image archive digest mismatch: {archive}") references[reference] = archive missing = sorted(set(expected_references) - set(references)) @@ -418,19 +458,21 @@ def _trusted_keys( keyring: Mapping[str, Any], *, now: datetime, + purpose: str, + label: str, ) -> dict[str, str]: _exact_keys( keyring, required={"schema_version", "purpose", "keys"}, - label="distribution keyring", + label=f"{label} keyring", ) if keyring.get("schema_version") != "1": - raise DistributionError("unsupported distribution keyring schema_version") - if keyring.get("purpose") != "govoplan-runtime-distribution": - raise DistributionError("distribution keyring has the wrong purpose") + raise DistributionError(f"unsupported {label} keyring schema_version") + if keyring.get("purpose") != purpose: + raise DistributionError(f"{label} keyring has the wrong purpose") values = keyring.get("keys") if not isinstance(values, list) or not values: - raise DistributionError("distribution keyring contains no keys") + raise DistributionError(f"{label} keyring contains no keys") trusted: dict[str, str] = {} for index, item in enumerate(values): key = _object(item, f"keyring.keys[{index}]") @@ -453,11 +495,11 @@ def _trusted_keys( pattern=KEY_ID, ) if key_id in trusted: - raise DistributionError("distribution keyring contains duplicate key ids") + raise DistributionError(f"{label} keyring contains duplicate key ids") if key.get("algorithm") != "ed25519": - raise DistributionError("distribution key must use ed25519") + raise DistributionError(f"{label} key must use ed25519") if key.get("status") not in {"active", "retired", "revoked"}: - raise DistributionError("distribution key has an invalid status") + raise DistributionError(f"{label} key has an invalid status") not_before = _datetime(key.get("not_before"), "key.not_before") expires = _datetime(key.get("expires_at"), "key.expires_at") public_key = key.get("public_key_pem") @@ -466,11 +508,11 @@ def _trusted_keys( or len(public_key.encode("utf-8")) > 8192 or "BEGIN PUBLIC KEY" not in public_key ): - raise DistributionError("distribution key has an invalid public key") + raise DistributionError(f"{label} key has an invalid public key") if key.get("status") == "active" and not_before <= now < expires: trusted[key_id] = public_key if not trusted: - raise DistributionError("distribution keyring has no currently active keys") + raise DistributionError(f"{label} keyring has no currently active keys") return trusted @@ -534,13 +576,17 @@ def _require_public_host(hostname: str) -> None: for value in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM) } except OSError as exc: - raise DistributionError(f"distribution host cannot be resolved: {hostname}") from exc + raise DistributionError( + f"distribution host cannot be resolved: {hostname}" + ) from exc if not addresses: raise DistributionError("distribution host resolved to no addresses") for value in addresses: address = ipaddress.ip_address(value) if not address.is_global: - raise DistributionError("distribution host resolves to a non-public address") + raise DistributionError( + "distribution host resolves to a non-public address" + ) def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str: @@ -553,7 +599,9 @@ def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str: try: opened = os.fstat(descriptor) if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes: - raise DistributionError(f"immutable artifact is invalid or too large: {path}") + raise DistributionError( + f"immutable artifact is invalid or too large: {path}" + ) while True: chunk = os.read(descriptor, 1024 * 1024) if not chunk: @@ -602,7 +650,11 @@ def _token( maximum: int, pattern: re.Pattern[str], ) -> str: - if not isinstance(value, str) or len(value) > maximum or pattern.fullmatch(value) is None: + if ( + not isinstance(value, str) + or len(value) > maximum + or pattern.fullmatch(value) is None + ): raise DistributionError(f"{label} is invalid") return value @@ -626,7 +678,11 @@ def _sha256(value: object, label: str) -> str: def _digest_image(value: object, label: str) -> str: - if not isinstance(value, str) or len(value) > 300 or DIGEST_IMAGE.fullmatch(value) is None: + if ( + not isinstance(value, str) + or len(value) > 300 + or DIGEST_IMAGE.fullmatch(value) is None + ): raise DistributionError(f"{label} must be an OCI image pinned by sha256") return value @@ -635,7 +691,12 @@ def _https_url(value: object, label: str) -> str: if not isinstance(value, str) or len(value) > 2048: raise DistributionError(f"{label} must be an HTTPS URL") parsed = urlsplit(value) - if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.username + or parsed.password + ): raise DistributionError(f"{label} must be an HTTPS URL without credentials") return value diff --git a/tools/deployment/govoplan_deploy/kubernetes.py b/tools/deployment/govoplan_deploy/kubernetes.py index 07b6326..182340b 100644 --- a/tools/deployment/govoplan_deploy/kubernetes.py +++ b/tools/deployment/govoplan_deploy/kubernetes.py @@ -10,6 +10,7 @@ import re from typing import Any, Mapping from urllib.parse import urlsplit +from .bundle import BACKUP_RUNTIME_ENV_KEYS from .model import InstallationSpec, image_is_digest_pinned @@ -55,6 +56,7 @@ _CONFIG_KEYS = ( "FILE_STORAGE_S3_BUCKET", "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + *BACKUP_RUNTIME_ENV_KEYS, ) _QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$") @@ -75,6 +77,8 @@ def render_kubernetes( secret_name: str = "govoplan-runtime", tls_secret_name: str = "govoplan-tls", ingress_class_name: str | None = None, + backup_required: bool = True, + backup_evidence: Mapping[str, object] | None = None, ) -> dict[str, Any]: """Render runtime roles only; shared state services stay externally managed.""" @@ -199,6 +203,8 @@ def render_kubernetes( environment, "MIGRATION", ), + backup_required=backup_required, + backup_evidence=backup_evidence, ), ] for pool in worker_pools: @@ -765,9 +771,28 @@ def _migration_job( secret_name: str, service_account: str, database_environment: Mapping[str, str], + backup_required: bool, + backup_evidence: Mapping[str, object] | None, ) -> dict[str, Any]: job_labels = {**labels, "app.kubernetes.io/component": "migration"} job_name = _name_with_suffix(name, f"migrate-{release_key}") + backup_annotations = { + "govoplan.add-ideas.de/backup-required": str(backup_required).lower(), + } + if backup_evidence is not None: + backup_annotations.update( + { + "govoplan.add-ideas.de/backup-evidence-sha256": str( + backup_evidence["evidence_sha256"] + ), + "govoplan.add-ideas.de/recovery-point": str( + backup_evidence["recovery_point_id"] + ), + "govoplan.add-ideas.de/restore-drill": str( + backup_evidence["restore_drill_id"] + ), + } + ) return { "apiVersion": "batch/v1", "kind": "Job", @@ -777,7 +802,7 @@ def _migration_job( "labels": job_labels, "annotations": { "govoplan.add-ideas.de/recovery-mode": "forward-recovery", - "govoplan.add-ideas.de/backup-required": "true", + **backup_annotations, "argocd.argoproj.io/sync-wave": "-1", }, }, diff --git a/tools/deployment/govoplan_deploy/planning.py b/tools/deployment/govoplan_deploy/planning.py index c163f9b..e964276 100644 --- a/tools/deployment/govoplan_deploy/planning.py +++ b/tools/deployment/govoplan_deploy/planning.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass +import hashlib import json import os from pathlib import Path @@ -18,6 +19,11 @@ from urllib.error import HTTPError, URLError from urllib.parse import urlsplit from urllib.request import Request, urlopen +from .backup_evidence import ( + MAX_BACKUP_EVIDENCE_BYTES, + MAX_BACKUP_KEYRING_BYTES, + verify_backup_evidence, +) from .bundle import ( BundlePaths, canonical_json, @@ -33,8 +39,11 @@ from .distribution import ( MAX_KEYRING_BYTES, MAX_MANIFEST_BYTES, DistributionError, + canonical_json as canonical_distribution_json, + decode_json_bytes, file_sha256, load_bounded_json, + read_bounded_bytes, verify_manifest, verify_manifest_binding, ) @@ -232,6 +241,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. ) checks.extend(_distribution_checks(spec, paths)) + checks.extend( + _backup_evidence_checks(spec, paths, receipt=_read_receipt(paths.receipt)) + ) values = read_env(paths.env) required = {"MASTER_KEY_B64", "DATABASE_URL"} @@ -350,6 +362,189 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. return tuple(checks) +def release_change_requires_backup( + spec: InstallationSpec, + receipt: Mapping[str, object], +) -> bool: + if spec.profile != "self-hosted" or not receipt: + return False + previous = receipt.get("release") + if not isinstance(previous, Mapping): + return True + desired = { + "channel": spec.release.channel, + "version": spec.release.version, + "manifest_sha256": spec.release.manifest_sha256, + "composition_sha256": spec.release.composition_sha256, + "api_image": spec.release.api_image, + "web_image": spec.release.web_image, + } + return any(previous.get(key) != value for key, value in desired.items()) + + +def verify_stored_backup_evidence( + spec: InstallationSpec, + paths: BundlePaths, + *, + receipt: Mapping[str, object], +) -> dict[str, object]: + verification = load_bounded_json( + paths.backup_verification, + maximum_bytes=64 * 1024, + ) + expected_fields = { + "schema_version", + "evidence_sha256", + "keyring_sha256", + "signature_key_id", + "verified_at", + "evidence_id", + "recovery_point_id", + "restore_drill_id", + "release_manifest_sha256", + "captured_at", + "expires_at", + "restore_started_at", + "restore_completed_at", + "measured_rpo_seconds", + "measured_rto_seconds", + "component_count", + } + if set(verification) != expected_fields or verification.get("schema_version") != 1: + raise DistributionError("backup verification receipt is malformed") + encoded_evidence = read_bounded_bytes( + paths.backup_evidence, + maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES, + ) + encoded_keyring = read_bounded_bytes( + paths.backup_keyring, + maximum_bytes=MAX_BACKUP_KEYRING_BYTES, + ) + evidence = decode_json_bytes(encoded_evidence, label="backup evidence") + keyring = decode_json_bytes(encoded_keyring, label="backup keyring") + if encoded_evidence != canonical_distribution_json(evidence): + raise DistributionError("stored backup evidence is not canonical JSON") + if encoded_keyring != canonical_distribution_json(keyring): + raise DistributionError("stored backup keyring is not canonical JSON") + evidence_digest = hashlib.sha256(encoded_evidence).hexdigest() + keyring_digest = hashlib.sha256(encoded_keyring).hexdigest() + if evidence_digest != verification.get("evidence_sha256"): + raise DistributionError("stored backup evidence digest has changed") + if keyring_digest != verification.get("keyring_sha256"): + raise DistributionError("stored backup keyring digest has changed") + previous_release = receipt.get("release") if receipt else None + expected_release: Mapping[str, object] = ( + previous_release + if isinstance(previous_release, Mapping) + else { + "channel": spec.release.channel, + "version": spec.release.version, + "manifest_sha256": spec.release.manifest_sha256, + "composition_sha256": spec.release.composition_sha256, + "api_image": spec.release.api_image, + "web_image": spec.release.web_image, + } + ) + summary = verify_backup_evidence( + evidence, + keyring, + installation_id=spec.installation_id, + profile=spec.profile, + release=expected_release, + ) + expected_summary = { + "signature_key_id": verification.get("signature_key_id"), + "evidence_id": verification.get("evidence_id"), + "recovery_point_id": verification.get("recovery_point_id"), + "restore_drill_id": verification.get("restore_drill_id"), + "captured_at": verification.get("captured_at"), + "expires_at": verification.get("expires_at"), + "restore_started_at": verification.get("restore_started_at"), + "restore_completed_at": verification.get("restore_completed_at"), + "measured_rpo_seconds": verification.get("measured_rpo_seconds"), + "measured_rto_seconds": verification.get("measured_rto_seconds"), + "component_count": verification.get("component_count"), + } + for field, expected in expected_summary.items(): + if summary.get(field) != expected: + raise DistributionError( + f"backup verification receipt does not match {field!r}" + ) + if expected_release.get("manifest_sha256") != verification.get( + "release_manifest_sha256" + ): + raise DistributionError("backup verification receipt has another release") + return { + **summary, + "evidence_sha256": evidence_digest, + "keyring_sha256": keyring_digest, + } + + +def _backup_evidence_checks( + spec: InstallationSpec, + paths: BundlePaths, + *, + receipt: Mapping[str, object], +) -> tuple[Check, ...]: + required = release_change_requires_backup(spec, receipt) + available = all( + path.is_file() + for path in ( + paths.backup_evidence, + paths.backup_keyring, + paths.backup_verification, + ) + ) + if not available: + return ( + Check( + "backup.migration_gate", + "error" + if required + else "warning" + if spec.profile == "self-hosted" + else "ok", + ( + "A release-changing migration has no verified coordinated backup evidence." + if required + else "No current coordinated backup evidence is adopted." + ), + ( + "Run verify-backup --adopt after an isolated restore drill." + if spec.profile == "self-hosted" + else "" + ), + ), + ) + try: + summary = verify_stored_backup_evidence( + spec, + paths, + receipt=receipt, + ) + except (DistributionError, OSError) as exc: + return ( + Check( + "backup.migration_gate", + "error" if required else "warning", + f"Coordinated backup evidence is invalid: {exc}", + "Adopt fresh signed evidence for the currently applied release.", + ), + ) + return ( + Check( + "backup.migration_gate", + "ok", + ( + "Release migration is backed by recovery point " + f"{summary['recovery_point_id']} and restore drill " + f"{summary['restore_drill_id']}." + ), + ), + ) + + def _ingress_configuration_checks( spec: InstallationSpec, paths: BundlePaths, diff --git a/tools/deployment/govoplan_deploy/recovery.py b/tools/deployment/govoplan_deploy/recovery.py index 1887e28..21cf2ce 100644 --- a/tools/deployment/govoplan_deploy/recovery.py +++ b/tools/deployment/govoplan_deploy/recovery.py @@ -29,6 +29,9 @@ _BUNDLE_FILES = ( "existing-proxy.json", "distribution-manifest.json", "distribution-keyring.json", + "backup-evidence.json", + "backup-keyring.json", + "backup-verification.json", "receipt.json", ) diff --git a/tools/deployment/sign-backup-evidence.py b/tools/deployment/sign-backup-evidence.py new file mode 100644 index 0000000..90e7c1d --- /dev/null +++ b/tools/deployment/sign-backup-evidence.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Sign and validate provider-produced GovOPlaN backup evidence.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +from pathlib import Path +import re +import stat +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from govoplan_deploy.backup_evidence import ( + MAX_BACKUP_EVIDENCE_BYTES, + load_backup_keyring, + verify_backup_evidence, +) +from govoplan_deploy.bundle import atomic_write +from govoplan_deploy.distribution import ( + canonical_json, + canonical_signed_payload, + load_bounded_json, +) + + +KEY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Sign a provider-produced backup/restore evidence document and " + "validate it against an independently managed public keyring." + ) + ) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--trusted-keyring", type=Path, required=True) + parser.add_argument( + "--signing-key", + action="append", + required=True, + metavar="KEY_ID=PRIVATE_PEM", + help="Ed25519 signer; may be repeated during key rotation.", + ) + parser.add_argument( + "--replace-signatures", + action="store_true", + help="Replace existing signatures instead of rejecting the input.", + ) + args = parser.parse_args() + + source = args.input.expanduser().resolve() + payload = load_bounded_json(source, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES) + existing = payload.get("signatures") + if existing not in (None, []) and not args.replace_signatures: + raise SystemExit("input already contains signatures; use --replace-signatures") + + signers = [_load_signer(value) for value in args.signing_key] + if len({key_id for key_id, _ in signers}) != len(signers): + raise SystemExit("duplicate signing key id") + payload["signatures"] = [] + signed = canonical_signed_payload(payload) + payload["signatures"] = [ + { + "key_id": key_id, + "algorithm": "ed25519", + "value": base64.b64encode(private_key.sign(signed)).decode("ascii"), + } + for key_id, private_key in signers + ] + + keyring = load_backup_keyring(args.trusted_keyring.expanduser().resolve()) + release = payload.get("release") + if not isinstance(release, dict): + raise SystemExit("input release must be an object") + verify_backup_evidence( + payload, + keyring, + installation_id=str(payload.get("installation_id") or ""), + profile=str( + _object(payload.get("deployment_subject"), "deployment_subject").get( + "profile" + ) + or "" + ), + release=release, + ) + encoded = canonical_json(payload) + output = args.output.expanduser().resolve() + atomic_write(output, encoded, mode=0o600) + print(f"Wrote {output}") + print(f"SHA256 {hashlib.sha256(encoded).hexdigest()}") + return 0 + + +def _load_signer(value: str) -> tuple[str, Ed25519PrivateKey]: + key_id, separator, raw_path = value.partition("=") + if not separator or KEY_ID.fullmatch(key_id) is None or not raw_path: + raise SystemExit("--signing-key must use KEY_ID=/path/to/private.pem") + path = Path(raw_path).expanduser().resolve() + mode = stat.S_IMODE(path.stat().st_mode) + if mode & 0o077: + raise SystemExit( + f"private signing key must not be group/world accessible: {path}" + ) + private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(private_key, Ed25519PrivateKey): + raise SystemExit(f"signing key is not Ed25519: {path}") + return key_id, private_key + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise SystemExit(f"input {label} must be an object") + return value + + +if __name__ == "__main__": + raise SystemExit(main())