Retain Caddy file capability at ingress boundary

.gitea/workflows/runtime-distribution.yml

@@ -39,6 +39,10 @@ on:
         description: Digest-pinned GreenMail image
         required: true
         type: string
+      binfmt_image:
+        description: Digest-pinned tonistiigi/binfmt image for arm64 CI execution
+        required: true
+        type: string
 
 jobs:
   publish-runtime:
@@ -53,13 +57,48 @@ jobs:
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
         with:
           node-version: "22"
+      - name: Validate immutable release inputs
+        env:
+          VERSION: ${{ inputs.version }}
+          PYTHON_IMAGE: ${{ inputs.python_image }}
+          NGINX_IMAGE: ${{ inputs.nginx_image }}
+          POSTGRES_IMAGE: ${{ inputs.postgres_image }}
+          REDIS_IMAGE: ${{ inputs.redis_image }}
+          LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }}
+          MANAGED_INGRESS_IMAGE: ${{ inputs.managed_ingress_image }}
+          GARAGE_IMAGE: ${{ inputs.garage_image }}
+          TEST_MAIL_IMAGE: ${{ inputs.test_mail_image }}
+          BINFMT_IMAGE: ${{ inputs.binfmt_image }}
+        run: |
+          python - <<'PY'
+          import os
+          import re
+
+          version = os.environ["VERSION"]
+          if re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?", version) is None:
+              raise SystemExit("version must be a SemVer value without a leading v")
+          image_pattern = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
+          for name in (
+              "PYTHON_IMAGE",
+              "NGINX_IMAGE",
+              "POSTGRES_IMAGE",
+              "REDIS_IMAGE",
+              "LOAD_BALANCER_IMAGE",
+              "MANAGED_INGRESS_IMAGE",
+              "GARAGE_IMAGE",
+              "TEST_MAIL_IMAGE",
+              "BINFMT_IMAGE",
+          ):
+              if image_pattern.fullmatch(os.environ[name]) is None:
+                  raise SystemExit(f"{name} must be an exact sha256 image reference")
+          PY
       - name: Use HTTPS for GovOPlaN repositories
         run: |
           git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan"
           git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "ssh://git@git.add-ideas.de/GovOPlaN/govoplan"
       - name: Bootstrap release sources
         working-directory: govoplan
-        run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --exclude-repo addideas-govoplan-website
+        run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website
       - name: Build release wheel roots and WebUI
         working-directory: govoplan
         run: |
@@ -162,6 +201,41 @@ jobs:
           WEB_DIGEST="sha256:$(sha256sum runtime-output/web-index.json | cut -d' ' -f1)"
           python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-api --index-digest "$API_DIGEST" --index runtime-output/api-index.json --output runtime-output/api-metadata.json
           python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-web --index-digest "$WEB_DIGEST" --index runtime-output/web-index.json --output runtime-output/web-metadata.json
+      - name: Resolve managed dependency platform images
+        working-directory: govoplan
+        env:
+          POSTGRES_IMAGE: ${{ inputs.postgres_image }}
+          REDIS_IMAGE: ${{ inputs.redis_image }}
+        run: |
+          docker buildx imagetools inspect "$POSTGRES_IMAGE" --raw > runtime-output/postgres-index.json
+          docker buildx imagetools inspect "$REDIS_IMAGE" --raw > runtime-output/redis-index.json
+          python tools/release/resolve-oci-platforms.py \
+            --repository "${POSTGRES_IMAGE%@*}" \
+            --index-digest "${POSTGRES_IMAGE##*@}" \
+            --index runtime-output/postgres-index.json \
+            --output runtime-output/postgres-metadata.json
+          python tools/release/resolve-oci-platforms.py \
+            --repository "${REDIS_IMAGE%@*}" \
+            --index-digest "${REDIS_IMAGE##*@}" \
+            --index runtime-output/redis-index.json \
+            --output runtime-output/redis-metadata.json
+      - name: Register arm64 execution for runtime smoke
+        working-directory: govoplan
+        env:
+          BINFMT_IMAGE: ${{ inputs.binfmt_image }}
+        run: docker run --privileged --rm "$BINFMT_IMAGE" --install arm64
+      - name: Exercise amd64 and arm64 runtime images
+        working-directory: govoplan
+        run: |
+          for ARCH in amd64 arm64; do
+            .runtime-build/bin/python tools/checks/runtime-image-smoke.py \
+              --api-metadata runtime-output/api-metadata.json \
+              --web-metadata runtime-output/web-metadata.json \
+              --postgres-metadata runtime-output/postgres-metadata.json \
+              --redis-metadata runtime-output/redis-metadata.json \
+              --platform "linux/$ARCH" \
+              --output "runtime-output/evidence/runtime-smoke-$ARCH.json"
+          done
       - name: Generate and sign distribution evidence
         working-directory: govoplan
         env:
@@ -202,21 +276,53 @@ jobs:
             --dependency "test_mail=$TEST_MAIL_IMAGE" \
             --output-directory runtime-output/evidence \
             --descriptor runtime-output/distribution-descriptor.json
-          python tools/release/generate-runtime-distribution.py \
+          .runtime-build/bin/python tools/release/generate-runtime-distribution.py \
             --descriptor runtime-output/distribution-descriptor.json \
             --signing-key "$SIGNING_KEY_ID=runtime-output/signing-key.pem" \
             --output runtime-output/distribution-manifest.json
           openssl pkeyutl -sign -inkey runtime-output/signing-key.pem -rawin \
             -in runtime-output/govoplan-deploy.pyz \
             -out runtime-output/govoplan-deploy.pyz.sig
-          sha256sum runtime-output/govoplan-deploy.pyz > runtime-output/govoplan-deploy.pyz.sha256
-          sha256sum runtime-output/distribution-manifest.json > runtime-output/distribution-manifest.json.sha256
+          (cd runtime-output && sha256sum govoplan-deploy.pyz > govoplan-deploy.pyz.sha256)
+          (cd runtime-output && sha256sum distribution-manifest.json > distribution-manifest.json.sha256)
           rm runtime-output/signing-key.pem
       - name: Verify the published bundle contract with the zipapp
         working-directory: govoplan
         env:
           VERSION: ${{ inputs.version }}
+          SIGNING_KEY_ID: ${{ secrets.RUNTIME_DISTRIBUTION_SIGNING_KEY_ID }}
         run: |
+          (cd runtime-output && sha256sum --check govoplan-deploy.pyz.sha256)
+          (cd runtime-output && sha256sum --check distribution-manifest.json.sha256)
+          .runtime-build/bin/python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          keyring = json.loads(
+              Path("runtime-output/distribution-keyring.json").read_text(encoding="utf-8")
+          )
+          key_id = os.environ["SIGNING_KEY_ID"]
+          matches = [item for item in keyring["keys"] if item.get("key_id") == key_id]
+          if len(matches) != 1 or matches[0].get("status") != "active":
+              raise SystemExit("runtime signing key is not uniquely active in the keyring")
+          Path("runtime-output/runtime-release-public.pem").write_text(
+              matches[0]["public_key_pem"], encoding="utf-8"
+          )
+          PY
+          openssl pkeyutl -verify -pubin \
+            -inkey runtime-output/runtime-release-public.pem -rawin \
+            -in runtime-output/govoplan-deploy.pyz \
+            -sigfile runtime-output/govoplan-deploy.pyz.sig
+          cp runtime-output/govoplan-deploy.pyz runtime-output/govoplan-deploy.tampered.pyz
+          printf '\0' >> runtime-output/govoplan-deploy.tampered.pyz
+          if openssl pkeyutl -verify -pubin \
+            -inkey runtime-output/runtime-release-public.pem -rawin \
+            -in runtime-output/govoplan-deploy.tampered.pyz \
+            -sigfile runtime-output/govoplan-deploy.pyz.sig >/dev/null 2>&1; then
+            echo "Tampered deployment bootstrap unexpectedly verified" >&2
+            exit 1
+          fi
           MANIFEST_SHA256="$(cut -d' ' -f1 runtime-output/distribution-manifest.json.sha256)"
           python runtime-output/govoplan-deploy.pyz init \
             --directory runtime-output/acceptance-install \
@@ -236,14 +342,17 @@ jobs:
           python tools/checks/managed-ingress-drill.py
           --caddy-image "$MANAGED_INGRESS_IMAGE"
           --load-balancer-image "$LOAD_BALANCER_IMAGE"
+          --probe-image "$(jq -r '.platforms["linux/amd64"]' runtime-output/api-metadata.json)"
       - name: Publish immutable Gitea release assets
         working-directory: govoplan
         env:
           VERSION: ${{ inputs.version }}
+          SOURCE_COMMIT: ${{ gitea.sha }}
           GITEA_RELEASE_TOKEN: ${{ secrets.GOVOPLAN_RELEASE_TOKEN }}
         run: |
           python tools/release/publish-runtime-release.py \
             --tag "v$VERSION" \
+            --target-commit "$SOURCE_COMMIT" \
             --title "GovOPlaN v$VERSION runtime distribution" \
             --asset runtime-output/govoplan-deploy.pyz \
             --asset runtime-output/govoplan-deploy.pyz.sig \
@@ -255,4 +364,6 @@ jobs:
             --asset runtime-output/evidence/api-sbom.cdx.json \
             --asset runtime-output/evidence/web-sbom.cdx.json \
             --asset runtime-output/evidence/api-provenance.json \
-            --asset runtime-output/evidence/web-provenance.json
+            --asset runtime-output/evidence/web-provenance.json \
+            --asset runtime-output/evidence/runtime-smoke-amd64.json \
+            --asset runtime-output/evidence/runtime-smoke-arm64.json

.gitea/workflows/runtime-ingress-drill.yml

@@ -0,0 +1,44 @@
+name: Runtime Ingress Drill
+
+on:
+  workflow_dispatch:
+    inputs:
+      caddy_image:
+        description: Digest-pinned Caddy image
+        required: true
+        type: string
+      load_balancer_image:
+        description: Digest-pinned HAProxy image
+        required: true
+        type: string
+      probe_image:
+        description: Digest-pinned amd64 GovOPlaN API image
+        required: true
+        type: string
+
+jobs:
+  managed-ingress:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+        with:
+          path: govoplan
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.12"
+      - name: Authenticate runtime image pull
+        env:
+          REGISTRY_USERNAME: ${{ secrets.GOVOPLAN_REGISTRY_USERNAME }}
+          REGISTRY_TOKEN: ${{ secrets.GOVOPLAN_REGISTRY_TOKEN }}
+        run: echo "$REGISTRY_TOKEN" | docker login git.add-ideas.de --username "$REGISTRY_USERNAME" --password-stdin
+      - name: Exercise the managed ingress boundary
+        working-directory: govoplan
+        env:
+          CADDY_IMAGE: ${{ inputs.caddy_image }}
+          LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }}
+          PROBE_IMAGE: ${{ inputs.probe_image }}
+        run: >-
+          python tools/checks/managed-ingress-drill.py
+          --caddy-image "$CADDY_IMAGE"
+          --load-balancer-image "$LOAD_BALANCER_IMAGE"
+          --probe-image "$PROBE_IMAGE"

docs/BACKUP_AND_RESTORE_EVIDENCE.md

@@ -0,0 +1,133 @@
+# Backup And Restore Evidence
+
+## Boundary
+
+`govoplan-deploy` verifies backup and restore evidence; it does not receive
+database, object-store, KMS, or orchestrator administration credentials and it
+does not create the backup. A provider-owned backup controller creates one
+coordinated recovery point, a separate drill runner restores it into an
+isolated target, and an evidence authority signs the resulting receipt.
+
+The application containers receive only a sanitized projection: evidence,
+recovery-point and drill identifiers, hashes, timestamps, component count, and
+measured RPO/RTO. Artifact locations, provider credentials, encryption-key
+references, the public trust keyring, and private signing keys remain in the
+deployment/evidence boundary.
+
+The machine-readable contracts are:
+
+- [`backup-evidence.schema.json`](backup-evidence.schema.json);
+- [`backup-evidence-keyring.schema.json`](backup-evidence-keyring.schema.json).
+
+One evidence document is bound to the installation id, deployment profile,
+topology subject, exact signed release manifest, image digests, and composition
+digest. It covers PostgreSQL, objects, protected configuration, and recoverable
+key custody at one recovery point. It contains references, never key material.
+
+## Production Sequence
+
+1. Establish the provider snapshot, application quiesce, or transaction
+   boundary and retain a hash of its fencing token.
+2. Capture PostgreSQL, object storage, protected deployment configuration, and
+   key-custody state within five minutes of that recovery point.
+3. Restore all four components into a target isolated from production write
+   endpoints and production queues.
+4. Start the exact immutable release named in the evidence, verify migration
+   heads, verify a deterministic manifest of representative object hashes, and
+   execute the documented semantic journey checks.
+5. Record actual data loss and elapsed recovery as measured RPO and RTO. A
+   measured RPO above the declared objective invalidates the evidence.
+6. Sign the canonical receipt using an evidence-authority Ed25519 key held
+   outside the application and deployment host. During key rotation, include
+   both accepted signatures.
+7. Transfer the evidence SHA-256 through an independent approved channel, then
+   verify and adopt it on the deployment host.
+
+Provider automation can sign and validate an unsigned receipt with:
+
+```sh
+python tools/deployment/sign-backup-evidence.py \
+  --input unsigned-backup-evidence.json \
+  --output backup-evidence.json \
+  --trusted-keyring backup-evidence-keyring.json \
+  --signing-key backup-authority-2026=/run/keys/backup-authority.pem
+```
+
+The private key file must be owner-only. The tool refuses an unexpected key
+type, an inactive/untrusted signer, malformed or partial evidence, stale
+recovery points, failed drill checks, mismatched releases, and non-canonical
+output.
+
+Adopt the result using the independently obtained digest:
+
+```sh
+python3 govoplan-deploy.pyz verify-backup \
+  --directory /srv/govoplan/default \
+  --evidence ./backup-evidence.json \
+  --evidence-sha256 "$APPROVED_BACKUP_EVIDENCE_SHA256" \
+  --trusted-keyring ./backup-evidence-keyring.json \
+  --adopt
+```
+
+Evidence is fresh for at most 24 hours and may declare an earlier expiry. Every
+self-hosted release identity change is conservatively treated as a migration
+boundary. `doctor`, Compose `apply`, and `render-kubernetes` fail closed when
+fresh evidence for the previously applied immutable release is unavailable.
+Compose verifies once before changing runtime state and again after API/worker
+quiescing immediately before migration. The exported Kubernetes migration Job
+is generated only after verification and is annotated with the sanitized
+evidence digest, recovery-point id, and drill id.
+
+## Provider Runbooks
+
+### PostgreSQL
+
+Use a managed transaction-consistent snapshot or a base backup plus retained
+WAL sufficient to reconstruct the declared point. Record the provider,
+protected artifact reference and digest, snapshot identity, and PostgreSQL LSN.
+The restore drill must connect only to the isolated database and must compare
+the resulting migration-head digest with the release expectation.
+
+### Object Storage
+
+Use provider snapshots/versioning or an immutable object copy. Build a sorted
+manifest containing object key, version, size, and content digest, then record
+its digest, object count, total bytes, provider version identity, and protected
+artifact reference. Verify representative objects from every owning module
+after restore. Single-node managed Garage is persistent but not highly
+available; copy its coordinated recovery material to an independent failure
+domain.
+
+### Configuration And Key Custody
+
+Back up the private installation bundle and external secret-manager bindings as
+an encrypted artifact. Record only its reference and digest. For KMS/HSM/vault
+state, record the provider keyset reference, version, and a successful
+recoverability assertion. Never put a key, recovery share, token, password, or
+credential-bearing URL in evidence. The isolated drill must prove that the
+restored release can decrypt representative protected content without
+exporting the key material into the report.
+
+## Ownership And Retention
+
+The deployment owner approves the RPO/RTO objectives. State-service owners
+operate backup capture and restoration. Module owners define representative
+objects and semantic checks. Security owns evidence-authority keys and
+revocation. Operations schedules drills and retains sanitized status.
+
+Retain backup artifacts for the approved legal/operational period and at least
+through the release's rollback window. Retain signed evidence, drill reports,
+and deletion receipts for the audit period. Disposal must remove every backup
+copy and provider version according to policy, then revoke or retire references
+without deleting the audit receipt. Cryptographic erasure is valid only when
+key-destruction evidence and provider-copy coverage are independently proven.
+
+## Failure Handling
+
+Missing components, component-time skew, stale or expired evidence, revocation,
+signature/key mismatch, changed stored files, release mismatch, failed semantic
+checks, or an RPO breach block migration. The deployment journal records the
+rejection without private provider details. If migration has not started, the
+operator may supply fresh evidence and retry. Once migration starts, recovery
+is explicitly forward-only until the verified coordinated recovery point is
+restored with its matching release.

docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md

@@ -97,6 +97,9 @@ The private installation directory contains:
 | `receipt.json` | Last successfully applied immutable identities |
 | `distribution-manifest.json` | Canonical signed runtime/image selection adopted by the installer |
 | `distribution-keyring.json` | Explicitly installed public trust anchor for runtime releases |
+| `backup-evidence.json` | Signed provider-neutral coordinated backup and isolated-restore receipt |
+| `backup-keyring.json` | Explicit public trust anchor for backup evidence authorities |
+| `backup-verification.json` | Sanitized local verification/adoption receipt |
 | `applied-state/` | Checksum-verified snapshot of the last healthy deployment bundle |
 | `operations/<id>/` | Private hash-chained deployment progress and recovery evidence |
 | `kubernetes.json` | Optional stateless multi-host Kubernetes export |
@@ -142,7 +145,54 @@ installation. Separate amd64/arm64 API and WebUI images are joined into OCI
 indexes and run as non-root identities. The release assets include CycloneDX
 application SBOMs, SLSA-style provenance, exact composition evidence, the
 single-file deployer, its detached Ed25519 signature, and a signed, expiring
-distribution manifest.
+distribution manifest. Evidence generation and signing run through the
+workflow's isolated release Python environment so their cryptographic tooling
+is explicit and independent of packages preinstalled in the Actions runner.
+The API image points Core at the migration scripts installed from the verified
+wheel under `/opt/govoplan/runtime/govoplan_core_runtime`; migrations therefore
+do not depend on a source checkout or the build host's Python installation
+scheme.
+Before publication, the exact amd64 and arm64 image manifests each run release
+migrations against the pinned PostgreSQL image, reach API and WebUI readiness
+as non-root/read-only processes, and complete a task through the pinned Redis
+image and packaged worker. Sanitized per-platform smoke receipts are retained
+as immutable release assets.
+PostgreSQL and Redis indexes are resolved to untagged platform-child digests
+before each smoke run. This keeps the evidence architecture-specific and
+avoids retargeting one local Docker tag between incompatible platforms.
+The CI host registers arm64 execution with an explicitly supplied,
+digest-pinned `tonistiigi/binfmt` image immediately before the smoke. This
+privileged helper is confined to the release runner and is never part of a
+GovOPlaN target deployment or its runtime image set.
+Because QEMU user-mode execution triggers Redis's arm64 host-kernel COW guard,
+the arm64 smoke suppresses only `ARM64-COW-BUG` while persistence, snapshots,
+and append-only files are disabled. Target Redis services never inherit this
+test-only option.
+The smoke also proves a bounded post-migration table contract and aborts as
+soon as a required container exits, rather than allowing a dead process to
+consume the full readiness timeout.
+
+Ingress acceptance streams generated configuration into Docker-managed
+volumes before starting the read-only containers. It therefore also works when
+an Actions job reaches a host or remote Docker daemon through a mounted socket;
+the drill never assumes that a job-container path is visible to that daemon.
+The drill allocates explicit loopback-only host ports and verifies Docker's
+host binding configuration, avoiding daemon-specific random-port shorthand
+behavior. Because an Actions job and deployment containers may be Docker
+siblings, functional HTTP/TLS checks run from the digest-pinned API image on
+the deployment network instead of assuming the Docker host is job-local.
+The dispatch-only `Runtime Ingress Drill` workflow exposes the same bounded
+check independently so ingress changes can be diagnosed before an immutable
+runtime publication; it accepts only digest-pinned Caddy, HAProxy, and API
+images and has no push trigger.
+The official Caddy binary carries the `NET_BIND_SERVICE` file capability. The
+managed-ingress container therefore drops every capability and adds back only
+`NET_BIND_SERVICE`; otherwise Linux rejects the binary at `execve` before its
+high-port configuration can start. `no-new-privileges`, a read-only root
+filesystem, and non-privileged container ports remain enforced.
+The bounded setup helper writes only generated public configuration as root so
+it can initialize a new volume; the actual HAProxy process retains the image's
+non-root identity and runs read-only with all capabilities dropped.
 
 The manifest contract is
 [`runtime-distribution-manifest.schema.json`](runtime-distribution-manifest.schema.json),
@@ -369,10 +419,12 @@ starts, recovery is forward-only unless an independently verified database
 backup is restored. See
 [Recovery And Rollback Guarantees](RECOVERY_AND_ROLLBACK_GUARANTEES.md).
 
-Production updates still need an operator-provided database backup/restore
-gate, database compatibility declaration, image signature verification, and
-deployment-specific drain policy. The deployment journal proves its own
-actions; it does not manufacture backup evidence.
+Production updates still need operator/provider-created coordinated backup and
+restore evidence, a database compatibility declaration, and a
+deployment-specific drain policy. The deployer now verifies and enforces the
+signed evidence before migration, but does not manufacture backups or receive
+provider administration credentials. See
+[Backup And Restore Evidence](BACKUP_AND_RESTORE_EVIDENCE.md).
 
 ## Stateless Kubernetes Runtime
 
@@ -391,7 +443,9 @@ The output includes a release-specific migration Job, database-head wait init
 containers, API readiness/liveness probes, rolling Deployments, Services, Pod
 disruption budgets, a tokenless ServiceAccount, and one fenced scheduler. Apply
 the named Secret through the cluster's secret manager and review ingress proxy
-CIDRs before deployment. Detailed rollout and scaling rules live in
+CIDRs before deployment. A release-changing export requires adopted backup
+evidence and carries only its sanitized digest and identifiers as Job
+annotations. Detailed rollout and scaling rules live in
 [Scaling And Multi-Host Deployment](SCALING_AND_MULTI_HOST_DEPLOYMENT.md).
 
 ## Recovery Commands
@@ -437,15 +491,27 @@ of the reviewed update recipe instead of a non-functional update button.
 
 ## Distribution Workflow
 
-The downloadable entry point is a release asset. Obtain the zipapp, detached
-signature, checksum, and trusted public keyring through independently
-authenticated paths before execution:
+The downloadable entry point is a reproducible release asset: sorted source
+paths, fixed ZIP metadata, fixed compression settings, and identical source
+bytes produce an identical zipapp regardless of checkout timestamps. Obtain the
+zipapp, detached signature, checksum, and trusted public keyring through
+independently authenticated paths before execution:
 
 ```sh
 curl --proto '=https' --tlsv1.2 --fail --location \
   https://git.add-ideas.de/GovOPlaN/govoplan/releases/download/vX.Y.Z/govoplan-deploy.pyz \
   --output govoplan-deploy.pyz
 sha256sum --check govoplan-deploy.pyz.sha256
+python3 - <<'PY'
+import json
+from pathlib import Path
+
+keyring = json.loads(Path("distribution-keyring.json").read_text())
+active = [key for key in keyring["keys"] if key["status"] == "active"]
+if len(active) != 1:
+    raise SystemExit("expected exactly one active runtime release key")
+Path("runtime-release-public.pem").write_text(active[0]["public_key_pem"])
+PY
 openssl pkeyutl -verify -pubin -inkey runtime-release-public.pem -rawin \
   -in govoplan-deploy.pyz -sigfile govoplan-deploy.pyz.sig
 python3 govoplan-deploy.pyz init

docs/INTERFACE_SURFACE_INVENTORY.md

@@ -15,7 +15,14 @@ machine-readable field, label, translation, route, API-reference, and module
 manifest evidence. This hand-maintained document remains the reviewed product
 interpretation and rollout ledger; generated evidence does not replace it.
 
-Snapshot refreshed: 2026-07-22.
+Snapshot refreshed: 2026-08-03.
+
+The generated snapshot contains 65 module manifests, 35 WebUI-contributing
+repositories, 40 statically declared module routes, 1,156 UI fields, and 836
+backend endpoints. All backend endpoints are classified and no stale endpoint
+declarations were found. The 234 endpoints without a static WebUI reference are
+kept visible as review evidence; they may intentionally serve workers, public
+clients, connectors, or external integrations.
 
 Evidence was read from tracked Git `HEAD` in the local GovOPlaN checkouts:
 
@@ -55,33 +62,50 @@ Inventory states:
 | Session/bootstrap state | `govoplan-core` `App.tsx` and `AppShell` | All browser sessions during bootstrap | Understand that session/platform state is loading; state contract | Unreviewed; core shell |
 | `/` authenticated redirect | `govoplan-core` chooses the first visible navigation destination | Authenticated; result depends on visible nav contributions | Enter the actor's first accessible service area; navigation behavior, not a content page | Unreviewed; focused-view/default-route work must preserve this fallback |
 | `/dashboard` fallback | `govoplan-core` `DashboardPage` only when the Dashboard module is absent | Authenticated; no route-specific scope in core | Cross-module starting point; dashboard | Unreviewed; compare with module dashboard before shared changes |
-| `/settings` | `govoplan-core` `SettingsPage` | Authenticated; contributed sections and integrations filter internally | Profile, UI/workspace preference, local connection, and user-scoped integration settings; configuration | Unreviewed; Core #225 program |
+| `/settings` | `govoplan-core` `SettingsPage` | Authenticated; contributed sections and integrations filter internally | Profile, UI/workspace preference, local connection, and user-scoped integration settings; configuration | Core-owned pattern migration complete in [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225), commit `fa32cca` |
 | Shell chrome | `AppShell`, `Titlebar`, `IconRail`, `BreadcrumbBar`, `HelpMenu`, language menu, unsaved-change provider | Public/authenticated variants; nav filtered later | Tenant/actor context, global navigation, help, language, session and maintenance state | Unreviewed; platform-owned prerequisite for focused views |
 
 ## Direct Module Route Contributions
 
-The access guard column reports only the route-level declaration in
-`module.ts`. Inner APIs and controls may impose additional checks.
+The access column summarizes only the route-level declaration in `module.ts`.
+Inner APIs and controls may impose additional checks. Public and compatibility
+routes are called out explicitly because they do not have the same manifest
+semantics as authenticated navigation routes.
 
-| Route | Owner / render evidence | Route-level access evidence | Primary task | Target archetype | Status / priority |
-| --- | --- | --- | --- | --- | --- |
-| `/admin` | `govoplan-access` `AdminPage` | Any core `adminReadScopes` | Administer system and tenant concerns assembled from module sections | Administration/configuration | Contributed; unreviewed; P1 under [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225) |
-| `/address-book` | `govoplan-addresses` `AddressBookPage` | `addresses:contact:read` | Browse and manage contacts, address books, and lists | Directory/list-detail | Contributed; unreviewed; P2 after Campaign |
-| `/calendar` | `govoplan-calendar` `CalendarPage` | `calendar:event:read` | Browse calendars/events and act on calendar data | Directory/list-detail | Contributed; metadata gap; unreviewed; P2 after Campaign |
-| `/campaigns` | `govoplan-campaign` `CampaignListPage` | `campaigns:campaign:read` | Find, compare, create, and open campaigns | List-detail entry | Pilot; P1 [Campaign #74](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/74) |
-| `/campaigns/:campaignId/*` | `govoplan-campaign` `CampaignResourceRoute` and `CampaignWorkspace` | `campaigns:campaign:read`, plus resource probe | Configure, review, send, and inspect one campaign/version | List-detail workspace containing edit, review, monitoring, and evidence surfaces | Pilot; P1 Campaign #74 |
-| `/operator` | `govoplan-campaign` `OperatorQueuePage` | `campaigns:campaign:read` and any of queue, control, retry, or reconcile | Monitor and intervene in campaign jobs through authority-specific controls | Monitoring/work queue | Pilot; durable queue controls delivered in [Campaign #78](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/78); #74 audit remains |
-| `/reports` | `govoplan-campaign` `AggregateReportsPage` | `campaigns:report:read` | Compare privacy-protected cross-campaign outcome totals without recipient detail, diagnostics, export, or drill-down | Aggregate reporting | Pilot; aggregate-reader surface delivered in [Campaign #80](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/80); #74 audit remains |
-| `/templates` | `govoplan-campaign` `TemplatesPage` | No route guard declared in `module.ts` | Browse/manage campaign templates | Directory/list-detail | Pilot audit; permission intent must be verified; P2 |
-| `/dashboard` | `govoplan-dashboard` `DashboardPage` | No route-specific scope | Assemble module-provided actionable widgets | Dashboard | Contributed; unreviewed; P2 |
-| `/docs` | `govoplan-docs` `DocsPage` | Docs read or system/tenant settings read scopes | Read configured, available, and evidence-aware documentation | Documentation directory/reference | Contributed; unreviewed; P1 [Docs #15](https://git.add-ideas.de/GovOPlaN/govoplan-docs/issues/15) after initial pattern content |
-| `/files` | `govoplan-files` `FilesPage` | `files:file:read` | Browse folders/files and perform managed-file work | Directory/explorer | Contributed; metadata gap; unreviewed; P2 after Campaign |
-| `/idm` | `govoplan-idm` `IdmPage` | Any IDM assignment/write or organization function-assign scope | Inspect and govern identity/function assignments | List-detail/configuration | Contributed; unreviewed; P2 |
-| `/mail` | `govoplan-mail` `MailboxPage` | `mail:mailbox:read` | Browse mailboxes and messages | Directory/list-detail | Contributed; metadata gap; unreviewed; P2 after Campaign |
-| `/notifications` | `govoplan-notifications` `NotificationCenterPage` | `notifications:notification:read` | Inspect and acknowledge notification state | List-detail/inbox | Contributed without a nav item or backend frontend metadata; navigation intent unknown; P2 discovery |
-| `/ops` | `govoplan-ops` `OpsPage` | Ops read or system/tenant settings read scopes | Inspect runtime health and readiness | Monitoring | Contributed; unreviewed; P2 |
-| `/organizations` | `govoplan-organizations` `OrganizationsPage` | Organization model/unit/function or admin settings read scopes | Model and inspect organizational structures/functions | Directory/list-detail | Contributed; unreviewed; P2 |
-| `/scheduling` | `govoplan-scheduling` `SchedulingPage` | `scheduling:schedule:read` | Plan and decide scheduling requests and availability | List-detail/guided decision | Contributed; metadata gap; unreviewed; P2 |
+| Routes | Owner | Route-level access | Primary archetype | Migration issue |
+| --- | --- | --- | --- | --- |
+| `/admin` | Access | Any declared administration/read scope | Administration/configuration host | Access pattern migration complete in [Access #19](https://git.add-ideas.de/GovOPlaN/govoplan-access/issues/19), commit `1409dbf`; shared host contract complete in [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225) |
+| `/address-book` | Addresses | `addresses:contact:read` | Governed source directory, contact/list detail, external-provider operation, governance facts, and reversible correction | Addresses pattern migration complete in [Addresses #23](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/23), commit `f9a7185` |
+| `/approvals` | Approvals | `approvals:workspace:read` | Work queue/guided decision | Approvals pattern migration complete in [Approvals #3](https://git.add-ideas.de/GovOPlaN/govoplan-approvals/issues/3), commit `24e9559` |
+| `/calendar` | Calendar | `calendar:event:read` | Full-height calendar workspace with filterable collection/agenda sidebar, continuous and bounded date views, guarded VEVENT and source editors, synchronized-source status, durable outbox recovery, and destructive remote-move evidence | Calendar pattern migration complete in [Calendar #22](https://git.add-ideas.de/GovOPlaN/govoplan-calendar/issues/22), commit `d7fd944` |
+| `/campaigns`, `/campaigns/:campaignId/*`, `/campaigns/queue`, `/campaigns/reports`, `/templates` | Campaign | Campaign read/report/control scopes; template route has no route guard | List-detail, guided review, monitoring, reporting | [Campaign #74](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/74) |
+| `/operator` | Campaign | Campaign read plus queue/control scope | Compatibility redirect to `/campaigns/queue` | Campaign #74; retire under the compatibility policy |
+| `/cases`, `/cases/:caseId` | Cases | `cases:case:read` | Governed case directory and detail workspace with guarded OCC lifecycle editor, provider-owned references, immutable timeline/history, and confirmed object-access editor | Cases pattern migration complete in [Cases #4](https://git.add-ideas.de/GovOPlaN/govoplan-cases/issues/4), commit `43b4cc8` |
+| `/committee` | Committee | `committee:workspace:read` | Governed workspace | Committee pattern migration complete in [Committee #2](https://git.add-ideas.de/GovOPlaN/govoplan-committee/issues/2), commit `e64af30` |
+| `/dashboard` | Dashboard | No route-specific scope | View-specific personal workspace with module/permission-filtered widget library, guarded four-column composition, nested widget settings, server/browser fallback, and optimistic layout persistence | Dashboard pattern migration complete in [Dashboard #3](https://git.add-ideas.de/GovOPlaN/govoplan-dashboard/issues/3), commit `da3947f` |
+| `/dataflow` | Dataflow | Pipeline read/admin | Governed library, guarded graph/constrained-SQL definition editor, typed node inspector, bounded intermediate preview, automation triggers, and durable run/deployment evidence | Dataflow pattern migration complete in [Dataflow #20](https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/20), commit `109ddcd` |
+| `/datasources` | Datasources | Catalogue read/source admin | Governed catalogue, staging preflight, optional-origin directory, authority editor, and immutable evidence | Datasources pattern migration complete in [Datasources #7](https://git.add-ideas.de/GovOPlaN/govoplan-datasources/issues/7), commit `6406ce7` |
+| `/distribution-lists` | Distribution Lists | List read/write/admin | Governed directory, immutable-revision editor, expansion preview, and evidence register | Distribution Lists pattern migration complete in [Distribution Lists #8](https://git.add-ideas.de/GovOPlaN/govoplan-dist-lists/issues/8), commit `6cdd804` |
+| `/docs` | Docs | Documentation or settings read | Documentation/reference | [Docs #15](https://git.add-ideas.de/GovOPlaN/govoplan-docs/issues/15) |
+| `/files` | Files | `files:file:read` | Directory/explorer | Files pattern migration complete in [Files #42](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/42), commit `d8ae506` |
+| `/forms` | Forms | `forms:definition:read` | Definition library/editor | Forms pattern migration complete in [Forms #4](https://git.add-ideas.de/GovOPlaN/govoplan-forms/issues/4), commit `e505536` |
+| `/forms-runtime`, `/forms-runtime/:instanceId` | Forms Runtime | Participate or workspace read | Guided form execution | Forms Runtime pattern migration complete in [Forms Runtime #5](https://git.add-ideas.de/GovOPlaN/govoplan-forms-runtime/issues/5), commit `07dd35b` |
+| `/idm` | IDM | Assignment, function-change, relationship, or organization scopes | Directory/governed change | IDM pattern migration complete in [IDM #12](https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/12), commit `d864317` |
+| `/mail`, `/mail/bounces` | Mail | Mailbox or bounce read/manage | Directory/explorer, operational evidence | Mail pattern migration complete in [Mail #20](https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/20), commit `7844d9c` |
+| `/notifications` | Notifications | `notifications:notification:read` | Inbox/list-detail with guarded recipient state, confirmed local cancellation/dispatch, and sanitized delivery evidence | Notifications pattern migration complete in [Notifications #4](https://git.add-ideas.de/GovOPlaN/govoplan-notifications/issues/4), commit `ad6a31f` |
+| `/ops` | Ops | Operations or settings read | Monitoring/evidence with contextual run, drain, readiness-blocker, and recovery guidance | Ops pattern migration complete in [Ops #4](https://git.add-ideas.de/GovOPlaN/govoplan-ops/issues/4), commit `2b32643` |
+| `/organizations` | Organizations | Model/unit/function or settings read | Directory/hierarchy editor | Organizations pattern migration complete in [Organizations #7](https://git.add-ideas.de/GovOPlaN/govoplan-organizations/issues/7), commit `97acfcb` |
+| `/portal` | Portal | `portal:service:read` | Service portal | [Portal #2](https://git.add-ideas.de/GovOPlaN/govoplan-portal/issues/2) |
+| `/postbox` | Postbox | `postbox:postbox:read` | Inbox/list-detail | Postbox pattern migration complete in [Postbox #26](https://git.add-ideas.de/GovOPlaN/govoplan-postbox/issues/26), commit `a97eb3b` |
+| `/projects` | Projects | `projects:project:read` | List-detail/project workspace | [Projects #2](https://git.add-ideas.de/GovOPlaN/govoplan-projects/issues/2) |
+| `/reporting`, `/reports` | Reporting | `reporting:definition:read` | Reporting/definition library | [Reporting #8](https://git.add-ideas.de/GovOPlaN/govoplan-reporting/issues/8) |
+| `/risk-compliance` | Risk Compliance | Workspace or sanctions read | Immutable source evidence, version-pinned screening, list-detail review, and revisioned assurance graph with explicit blockers and consequences | Risk Compliance pattern migration complete in [Risk Compliance #8](https://git.add-ideas.de/GovOPlaN/govoplan-risk-compliance/issues/8), commit `24d80a6` |
+| `/scheduling` | Scheduling | `scheduling:schedule:read` | List-detail/guided decision | Scheduling pattern migration complete in [Scheduling #8](https://git.add-ideas.de/GovOPlaN/govoplan-scheduling/issues/8), commit `c17cbda` |
+| `/scheduling/public/:requestId/:token` | Scheduling | Public signed token | Public participation | Scheduling #8 complete in `c17cbda` |
+| `/search` | Search | `search:result:read` | Search overlay/results | [Search #4](https://git.add-ideas.de/GovOPlaN/govoplan-search/issues/4) |
+| `/templates` | Templates | Template read/write/publish/render/admin | Governed library, immutable-revision editor, compatibility preview, and render evidence | Templates pattern migration complete in [Templates #5](https://git.add-ideas.de/GovOPlaN/govoplan-templates/issues/5), commit `72fafa2` |
+| `/voting` | Voting | `voting:ballot:read` | Governed ballot workspace | Voting pattern migration complete in [Voting #1](https://git.add-ideas.de/GovOPlaN/govoplan-voting/issues/1), commit `2625990` |
+| `/workflow` | Workflow | Definition read or instance admin | Graph editor/execution evidence | [Workflow #15](https://git.add-ideas.de/GovOPlaN/govoplan-workflow/issues/15) |
 
 ## Manifest And Runtime Route Alignment
 
@@ -91,28 +115,27 @@ loading reason about the configured interface without executing module UI code.
 is recorded here as an evidence gap; this inventory does not infer whether each
 gap is intentional.
 
-| Module | `module.ts` routes | Backend manifest frontend routes | Backend nav alignment | Result |
-| --- | --- | --- | --- | --- |
-| Access | `/admin` | `/admin` | Aligned | Described |
-| Addresses | `/address-book` | `/address-book` | Aligned | Described |
-| Admin | No direct route; `admin.sections` | None | Not applicable | Composed surface |
-| Audit | No direct route; `admin.sections` | None | Not applicable | Composed surface |
-| Calendar | `/calendar` | None | `/calendar` nav exists | Metadata gap |
-| Campaign | Five routes | None | Four top-level nav items exist | Metadata gap; wildcard resource route is also undescribed |
-| Dashboard | `/dashboard` | `/dashboard` | Aligned | Described |
-| Docs | `/docs` | `/docs` | Aligned | Described |
-| Files | `/files` | None | `/files` nav exists | Metadata gap |
-| IDM | `/idm` | `/idm` | Aligned | Described |
-| Mail | `/mail` | None | `/mail` nav exists | Metadata gap |
-| Notifications | `/notifications` | No frontend metadata | No nav item | Metadata and discovery gap |
-| Ops | `/ops` | `/ops` | Aligned | Described |
-| Organizations | `/organizations` | `/organizations` | Aligned | Described |
-| Policy | No direct route; `admin.sections` | None | Not applicable | Composed surface |
-| Scheduling | `/scheduling` | None | `/scheduling` nav exists | Metadata gap |
+The generated comparison is aligned for all authenticated canonical routes.
+Two deliberate exceptions remain visible:
 
-Before a release claims a complete configured-system route inventory, add a
-contract check or explicit exceptions so executable routes and manifest
-metadata cannot silently diverge.
+- Campaign contributes `/operator` as a compatibility redirect for saved View
+  projections; its canonical and manifest-declared destination is
+  `/campaigns/queue`.
+- Scheduling contributes `/scheduling/public/:requestId/:token` through the
+  separate `publicRoutes` contract. Authenticated manifest routes intentionally
+  do not describe public signed-token entry points yet.
+
+Admin, Audit, Policy, Tenancy, and Views contribute composed administration or
+settings surfaces rather than direct routes. Their migration issues are
+[Admin #8](https://git.add-ideas.de/GovOPlaN/govoplan-admin/issues/8),
+[Audit #8](https://git.add-ideas.de/GovOPlaN/govoplan-audit/issues/8),
+[Policy #11](https://git.add-ideas.de/GovOPlaN/govoplan-policy/issues/11),
+[Tenancy #6](https://git.add-ideas.de/GovOPlaN/govoplan-tenancy/issues/6), and
+[Views #2](https://git.add-ideas.de/GovOPlaN/govoplan-views/issues/2).
+
+Release evidence must continue to run the generated inventory and manifest
+shape checks so new executable routes, public routes, aliases, and composed
+surfaces cannot silently diverge from their declared metadata.
 
 ## Composed Surfaces And Extension Points
 
@@ -121,25 +144,108 @@ enabled and the actor passes the declared filters.
 
 | Host surface | Contributor and evidence | Contributed regions/actions | Pattern implication | Audit |
 | --- | --- | --- | --- | --- |
-| `/admin` | Access host (`AdminPage`) | System tenants/users/roles, tenant users/groups/roles/API keys/settings, function-role mappings, user/group mail and file connector scopes | One stable admin information architecture must contain both host-owned and contributed sections | Unreviewed; P1 Core #225 |
-| `/admin` | `govoplan-admin` `admin.sections` | Overview; system settings; configuration changes; configuration packages; role/group templates; module management | Configuration, guided operations, review/preflight, consequence | In progress under Core #225; surface-level evidence still needed |
-| `/admin` | `govoplan-audit` `admin.sections` | System audit; tenant audit | Evidence/provenance and reporting | Unreviewed |
-| `/admin` | `govoplan-files` `admin.sections` and `files.connectors` | System and tenant file connections plus scoped connector managers used by Access | Adaptive configuration, discovery/test, policy and credentials | First migration family in Core #225; verification incomplete in this inventory |
-| `/admin` | `govoplan-organizations` `admin.sections` | Tenant organization settings | Configuration/list-detail | Unreviewed |
-| `/admin` | `govoplan-policy` `admin.sections` | System, tenant, group, and user retention | Effective value, source/provenance, consequential configuration | Unreviewed; Core #225 phase 4 |
-| `/admin` and `/settings` | `govoplan-mail` `mail.profiles` | System/tenant/group/user mail profile and policy managers | Same server/credential/policy grammar as file connectors | Unreviewed; Core #225 mail migration |
-| `/settings` | Core host | Profile; interface; workspace; local connection | Personal configuration with adaptive forms and immediate feedback | Unreviewed |
-| `/settings` | Files and Mail named capabilities | User-scoped file connections and mail profiles/policy | Optional integration regions disappear cleanly when capability absent | Unreviewed |
+| `/admin` | Access host (`AdminPage`) | System tenants/users/roles, tenant users/groups/roles/API keys/settings, function-role mappings, user/group mail and file connector scopes | One stable admin information architecture must contain both host-owned and contributed sections | Pattern migration, contextual help, explained permission/protection states, optional-module blockers, localization, and focused evidence complete in Access #19 (`1409dbf`); Core #225 shared host contract complete |
+| `/admin` | `govoplan-admin` `admin.sections` | Overview; system settings; configuration changes; configuration packages; role/group templates; module management | Configuration, guided operations, review/preflight, consequence | Pattern migration, contextual help, explained permission/protection/applicability states, guarded consequential actions, localization, and focused evidence complete in Admin #8 (`d428f33`) |
+| `/admin` | `govoplan-tenancy` `admin.sections` | System tenant registry and active-tenant settings | Administration directory, effective configuration, lifecycle consequence | Pattern migration, contextual help, explained permission/lifecycle/system-policy states, dirty-state guards, localization, and focused evidence complete in Tenancy #6 (`e76fe16`) |
+| `/admin` | `govoplan-audit` `admin.sections` | System audit; tenant audit | Evidence/provenance and reporting | Pattern migration, localized evidence projection, contextual help, and focused tests complete in Audit #8 (`6d3fcc1`) |
+| `/admin` | `govoplan-files` `admin.sections` and `files.connectors` | System and tenant file connections plus scoped connector managers used by Access | Adaptive configuration, discovery/test, policy and credentials | Pattern migration, contextual help, blocker explanations and focused evidence complete in Files #42 (`d8ae506`) |
+| `/admin` | `govoplan-organizations` `admin.sections` | Tenant organization settings | Configuration/list-detail | Pattern migration, tenant-owned provenance, contextual help, guarded settings/editor drafts, explained permission states, localization and focused evidence complete in Organizations #7 (`97acfcb`) |
+| `/admin` | `govoplan-policy` `admin.sections` | System, tenant, group, and user retention | Effective value, source/provenance, consequential configuration | Pattern migration complete in Policy #11 (`f964ed7`) with Core editor contract `fa32cca` |
+| `/admin` and `/settings` | `govoplan-mail` `mail.profiles` | System/tenant/group/user mail profile and policy managers | Same server/credential/policy grammar as file connectors | Pattern migration, contextual help, policy/target/permission blockers and focused evidence complete in Mail #20 (`7844d9c`; shared test-reason contract Core `2d0551a`) |
+| `/settings` | Core host | Profile; interface; workspace; local connection | Personal configuration with adaptive forms and immediate feedback | Pattern migration complete in Core #225 (`fa32cca`) |
+| `/settings` | Files and Mail named capabilities | User-scoped file connections and mail profiles/policy | Optional integration regions disappear cleanly when capability absent | Files #42, Mail #20 and Core #225 complete |
+| `/admin` and `/settings` | `govoplan-views` `admin.sections`, `settings.sections`, and `views.runtime` | System/tenant definition and assignment editors, personal/group editors, global selector | Versioned presentation projection with inheritance, lockout safeguards, optional directory targets, and no authorization effect | Pattern migration, contextual help, localized selector/editor, guarded drafts, explained inherited/permission/capability states, and focused evidence complete in Views #2 (`c125f33`) |
 | `/settings` | `govoplan-notifications` `settings.sections` | Notification preferences | Personal configuration | Unreviewed |
 | `/dashboard` | Dashboard host and `dashboard.widgets` | Installed-modules widget; Ops health widget when Ops contributes it | Widget ordering, staleness, permissions, destination behavior | Unreviewed |
-| `/organizations` | IDM `organizations.functionActions` | Action leading to assignment view filtered by IDM scopes | Cross-module context action through explicit capability | Unreviewed |
+| `/organizations` | IDM `organizations.functionActions` | Action leading to assignment view filtered by IDM scopes | Cross-module context action through explicit capability | IDM pattern migration complete in IDM #12 (`d864317`) |
 | Campaign attachments/import | Files `files.fileExplorer` | Folder tree, managed chooser, file listing/pattern resolution/sharing | Optional domain composition without sibling-private imports | Pilot audit under Campaign #74 |
-| Campaign review/send | Mail runtime `mail.devMailbox` | Mock-mail verification when backend advertises runtime capability | Optional review stage with unavailable/optional states | Pilot audit under Campaign #63/#62 |
+| Campaign review/send | Mail runtime `mail.devMailbox` | Mock-mail verification when backend advertises runtime capability | Optional review stage with unavailable/optional states | Explicit intervention and review-progress vocabulary delivered in Campaign #63; send modes/progress delivered in #62/#79 |
 
 Other named capability exports (`files.connectors`, `organizations.functionPicker`,
 and mail profile validation) are contracts consumed inside the composed surfaces
 above; they are not independent routes.
 
+## Core Configuration Surface Map
+
+Core #225 now supplies and verifies the platform-owned configuration contract.
+The durable Core inventory is
+`govoplan-core/docs/INTERFACE_PATTERN_MIGRATION.md`.
+
+| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
+| --- | --- | --- | --- | --- |
+| `/settings` (`SettingsPage`) | Change personal profile, interface/workspace preferences, or local development connection | Two-zone typed settings workspace | Changes are user-scoped; save and test actions distinguish clean, busy, and active states | Contextual help, unsaved guard, typed controls and keyboard-explainable disabled actions in Core `fa32cca` |
+| Reusable credentials (`CredentialEnvelopeManager`) | Compare and configure scoped reusable authentication material | Repeated administration plus adaptive create/edit | Secret values are write-only; permission and missing-owner states block mutation explicitly; deletion can break dependent connections | Actionable blocker, stable row actions, typed references, unsaved guard and shared destructive confirmation |
+| Retention (`RetentionPolicyManagement`) | Inspect effective retention and narrow permitted local values | Effective-policy editor | Parent locks, source paths and write authority control whether sensitive evidence can be retained | Typed narrowing controls, source-path help, lock/target/permission blockers and clean/loading/save reasons |
+| Shared configuration primitives | Compose module-owned settings without sibling-private imports | Platform behavior contract | Consequence, focus, help, async, confirmation and permission semantics remain consistent | Core component suites, 121 module-system tests and full-product type/build/bundle gates |
+
+No primary Core configuration flow requires raw JSON. Expert JSON remains
+limited to diagnostics, interchange, conflict evidence, or read-only inspection.
+
+## Policy Surface Map
+
+Policy #11 verifies the four composed retention sections. The durable
+module-level inventory is
+`govoplan-policy/docs/INTERFACE_PATTERN_MIGRATION.md`.
+
+| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
+| --- | --- | --- | --- | --- |
+| System retention | Set the instance ceiling and run retention | Effective-policy editor plus destructive operation | An applied run can irreversibly redact/delete retained content; dry-run and applied evidence remain distinct | Core source-path/lock contract, permission and busy reasons, shared confirmation, typed/filterable outcome grid and audit-oriented wording |
+| Tenant retention | Narrow the inherited system ceiling | Effective-policy editor | Tenant policy cannot silently loosen its parent | Core typed controls, effective path and parent-lock explanation |
+| Group and user retention | Select an authorized target and narrow inherited policy | Targeted effective-policy editor | Selection exposes only bounded account/group labels; no retained content is returned | Delta-backed target loading, retry, missing-target blocker and responsive shared admin composition |
+
+Automated evidence for Policy `f964ed7` comprises 50 backend/manifest tests,
+the Policy interface structural gate, 65 manifest-shape checks, and the
+full-product TypeScript/Vite build with structural localization, theme and
+bundle-budget gates. Policy uses no sibling-private imports.
+
+## Files Surface Map
+
+Files #42 classifies and verifies the complete Files-owned route and composition
+boundary. The durable module-level inventory is
+`govoplan-files/docs/INTERFACE_PATTERN_MIGRATION.md`.
+
+| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
+| --- | --- | --- | --- | --- |
+| `/files` (`FilesPage`) | Browse spaces/folders and repeatedly act on current content | Full-height directory/explorer | Navigation is low consequence; upload, synchronize, move, copy and share are medium; delete is high | Stable two-pane composition, contextual help, selection/permission/state-specific disabled reasons, shared confirmation and responsive collapse |
+| Upload/archive, transfer, rename and connector-import dialogs | Supply, validate and review one bounded change | Adaptive create/edit or guided import | Writes managed content and may resolve conflicts or import untrusted bytes | Shared dialogs/drop zone, bounded archive preflight, conflict review, explicit confirmation and no browser-native confirmation |
+| Share/access explanation | Inspect or change who can use a resource | Review/decision | Grants can disclose content; delete/revoke changes access | Shared access explanation, action components and destructive confirmation; backend redaction remains authoritative |
+| File connector tree and connection/credential dialogs | Compare and configure external endpoints and reusable credentials | Administration plus adaptive create/edit | Endpoint, secret and capability changes can enable remote access | Shared connection tree/forms/advanced panel, endpoint discovery and login test, unsaved-change guard, read-only deployment provenance and actionable disabled reasons |
+| Connector policy card | Narrow effective connector use | Effective-policy editor | Inherited deny/allow rules affect lower scopes | Typed selectors, deny-precedence warning, effective sources, contextual admin help and permission blocker |
+| `files.widget.spaces` | See available spaces and enter Files | Dashboard widget | Space/provider names remain permission-filtered | Shared loading, alert and status components; bounded configuration and refresh |
+| `files.fileExplorer` capability | Select a governed managed snapshot for another module | Directory chooser | Exact file/version becomes another module's governed input | Capability-only composition, no sibling-private import, stable chooser/confirmation and exact snapshot evidence |
+
+Automated evidence for commit `d8ae506` comprises 104 Files backend tests,
+three focused Files WebUI structure tests, the full-product TypeScript/Vite
+build, structural localization audit, theme contract and bundle budget. Shared
+Dialog and disabled-tooltip behavior provide focus entry/return and
+keyboard-reachable explanations; responsive source order is guarded at 1050 px
+and 760 px. Secrets are not returned to the WebUI, and JSON remains only an
+advanced provider-compatibility escape hatch rather than the primary editor.
+
+## Mail Surface Map
+
+Mail #20 classifies and verifies the complete Mail-owned route and composition
+boundary. The durable module-level inventory is
+`govoplan-mail/docs/INTERFACE_PATTERN_MIGRATION.md`.
+
+| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
+| --- | --- | --- | --- | --- |
+| `/mail` (`MailboxPage`) | Browse an authorized provider mailbox without changing it | Full-height directory/explorer | Message metadata and content are private; every provider read is bounded and non-mutating | Stable three-pane composition, contextual help, explicit no-profile blocker, refresh reasons, keyboard rows, paging and responsive collapse |
+| Mail profile tree and profile/server/credential dialogs | Compare and configure reusable transport identities | Administration plus guided/adaptive create/edit | Endpoint and credential changes can enable external effects | Shared connection tree/dialog/stage rail/forms, focused hierarchy editors, unsaved guard, connection tests, permission/target blockers and disabled-save reasons |
+| Mail policy card | Narrow profile visibility, lower-scope definitions and transport/address patterns | Effective-policy editor | Inherited allow/deny rules affect delivery and lower scopes | Typed selectors and controls, effective source path, lock/read-only blocker, dirty-save state and contextual admin help |
+| `/mail/bounces` watcher table | Configure and explicitly scan bounded IMAP evidence sources | Operational administration | Provider access changes durable source cursors and evidence | Shared grid/status/loading/alerts, actionable no-profile and busy states, field help and stable row actions |
+| `/mail/bounces` observations and watcher removal | Review sanitized delivery outcomes or stop future scans | Evidence/reporting plus destructive confirmation | Recipient diagnostics are sensitive; watcher removal retains existing evidence | Bounded sanitized rows and shared confirmation with retained-evidence consequence |
+| `mail.profiles` and reference-selector capabilities | Select/validate Mail-owned transport from another module | Governed capability composition | A selected identity can perform external effects | Stable references, Mail-owned authorization/secret resolution, no sibling-private imports and clean optional absence |
+
+Automated evidence for Mail commit `7844d9c` and Core commit `2d0551a`
+comprises 114 Mail backend tests, Mail's focused UI/model/structure suite, the
+Core shared mail-component suite, 65 manifest-shape checks and the full-product
+TypeScript/Vite build with structural localization, theme and bundle-budget
+gates. Shared Dialog and disabled-tooltip behavior provides focus containment,
+return and keyboard-reachable explanations. Responsive source order is guarded
+at 1250 px, 900 px and 760 px. Passwords remain write-only, mailbox responses
+are bounded, and bounce evidence excludes raw provider messages.
+
 ## Campaign Pilot Surface Map
 
 Campaign is detailed first because it exercises almost every archetype. The
@@ -166,7 +272,7 @@ prove that the composition or states satisfy the pattern.
 | Campaign settings (`GlobalSettingsPage` settings view) | Configure campaign behavior | Adaptive configuration | Can alter validation/build/send behavior | #74 audit |
 | Mail policy (`MailSettingsPage` policy view) | Inspect/override effective mail policy | Effective policy/provenance editor | Inheritance and locks affect allowed delivery | #74; Core #225 policy pattern |
 | Campaign policy (`GlobalSettingsPage` policy view) | Inspect/override campaign policy | Effective policy/provenance editor | Inheritance, actor authority, and blocked edits | #74; Core #225 policy pattern |
-| Review/send (`ReviewSendPage`) | Validate, build, mock-test, confirm/send, inspect results | Guided review/decision plus durable progress | External communication, bounded synchronous execution, persisted queue mode, partial effects, retries, evidence | Bounded synchronous and explicit/persisted queued modes delivered in [#62](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/62) and [#79](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/79); [#63](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/63) wording and #74 audit remain |
+| Review/send (`ReviewSendPage`) | Validate, build, mock-test, confirm/send, inspect results | Guided review/decision plus durable progress | External communication, bounded synchronous execution, persisted queue mode, partial effects, retries, evidence | Blocking/non-blocking interventions and reviewed/remaining evidence delivered in [#63](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/63); bounded synchronous and explicit/persisted queued modes delivered in [#62](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/62) and [#79](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/79); #74 audit remains |
 | Message and attachment detail overlays | Inspect one built/mock message and its attachment links | Stable detail/review dialog | Personal data, exact outbound content, reviewed state | Delivered and verified in [#59](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/59) and [#73](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/73) |
 | Campaign report (`CampaignReportPage`) | Filter and inspect delivery outcomes | Reporting/list-detail | Partial, failed, explicitly excluded/skipped, SMTP/IMAP outcomes and retries | Server-owned filtering and counts delivered in [#65](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/65) with the full-result DataGrid contract from [Core #263](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/263); excluded semantics in [#66](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/66) |
 | Audit (`CampaignAuditPage`) | Inspect campaign evidence/history | Provenance timeline/report | Actor/action/effect trace | #74 audit |
@@ -182,24 +288,26 @@ The five review stages currently named in code are `Validate and inspect`,
 results`. Campaign #63 owns the intervention and status vocabulary; Workflow is
 not required to define or implement it.
 
-## Repositories Without A WebUI Route Contribution
+## Repositories Without A WebUI Package
 
-The following local repositories contain a backend manifest but no
-`webui/src/module.ts` at this snapshot:
+The generated manifest snapshot reports no WebUI package for:
 
-`govoplan-approvals`, `govoplan-assets`, `govoplan-booking`,
-`govoplan-certificates`, `govoplan-committee`, `govoplan-consultation`,
-`govoplan-contracts`, `govoplan-dist-lists`, `govoplan-evaluation`,
-`govoplan-facilities`, `govoplan-forms-runtime`, `govoplan-grants`,
-`govoplan-helpdesk`, `govoplan-identity`, `govoplan-inspections`,
-`govoplan-tickets`, `govoplan-learning`, `govoplan-permits`,
-`govoplan-poll`, `govoplan-procurement`, `govoplan-records`,
-`govoplan-resources`, `govoplan-rest`, `govoplan-risk-compliance`,
-`govoplan-soap`, `govoplan-tenancy`, and `govoplan-transparency`.
+`govoplan-assets`, `govoplan-booking`, `govoplan-certificates`,
+`govoplan-connectors`, `govoplan-consultation`, `govoplan-contracts`,
+`govoplan-decisions`, `govoplan-encryption`, `govoplan-evaluation`,
+`govoplan-facilities`, `govoplan-grants`, `govoplan-helpdesk`,
+`govoplan-identity`, `govoplan-identity-trust`, `govoplan-inspections`,
+`govoplan-learning`, `govoplan-mandates`, `govoplan-parties`,
+`govoplan-permits`, `govoplan-poll`, `govoplan-procurement`,
+`govoplan-records`, `govoplan-resources`, `govoplan-rest`,
+`govoplan-services`, `govoplan-soap`, `govoplan-tickets`,
+`govoplan-transparency`, `govoplan-wiki`, and `govoplan-workflow-engine`.
 
-This is only negative route evidence. It does not classify the backend module's
-maturity or decide that it needs a WebUI. Connector-only, capability-only, or
-backend-only modules may remain intentionally headless.
+Tenancy does provide composed administration surfaces despite having no direct
+route. This section is only negative package evidence; connector-only,
+capability-only, runtime-only, and backend-only modules may intentionally remain
+headless. A new WebUI should be created only for a concrete user task, not to
+make every module symmetrical.
 
 ## Rollout Matrix
 
@@ -208,17 +316,17 @@ backend-only modules may remain intentionally headless.
 | 0 | Product grammar and route inventory | Doctrine, ledger, layout rules, module contract, current route sources | One reconciled pattern language and evidence inventory | Meta [#11](https://git.add-ideas.de/GovOPlaN/govoplan/issues/11) | Docs links/diff checks; issue/wiki sync after integration | Initial slice in this document |
 | 1 | Campaign baseline integration | Recipient-editor WIP and tracker state have been reconciled with remote `main` | Integrated, testable baseline before migration claims | Campaign #67 and tracker cleanup | Backend and focused WebUI suites; issue evidence | Complete 2026-07-22 |
 | 2 | Campaign previews/details | Stable shared dialog with bounded scrolling and fixed responsive preview workspace | Stable header/body/footer, accessible long-content detail | Campaign #59 and #73 | Review-preview and overlay structure tests | Complete 2026-07-22 |
-| 3 | Campaign review/interventions | Five domain-owned stages with unresolved intervention language | Clear stages, outcomes, blockers, next actor/action, reviewed evidence | Campaign #63 | State matrix behavior/accessibility tests and agreed vocabulary | P1 needs product wording decision |
+| 3 | Campaign review/interventions | Five domain-owned stages use central blocker and guided-review primitives; validation/build warnings name action, actor, and destination; hard blockers, individual review, and group review remain distinct; reviewed/remaining counts survive reload through build-bound review evidence | Clear stages, outcomes, blockers, next actor/action, reviewed evidence | Campaign #63 | `reviewProgress` state tests, shared-component structure contract, TypeScript build, configured-system help topic, and Campaign documentation tests | Complete 2026-08-03 (`d635f3a`; Core primitives and contextual help `b823a22`) |
 | 4 | Campaign send/progress | A hard deployment ceiling bounds synchronous delivery; the selected synchronous, worker-queue, or database-queue mode is explicit and persisted; progress and recovery survive navigation; immediate-send response and audit evidence are allowlisted | Pre-send mode/consequence plus durable leave/return progress, retry and reconciliation without recipient/provider leakage | Campaign #62 and #79 | Boundary/concurrency/preflight, async selection, persisted mode, sanitized response/audit, partial/failure/retry and reload/return tests | Complete 2026-07-22 (`7e16603`, `60efd1c`, `62a6879`, `b0282eb`, `f095a3e`) |
 | 5 | Campaign report filtering | Core DataGrid distinguishes client/full-result from server-owned queries; Campaign applies filter/sort/count before pagination and synchronizes count shortcuts with the grid query | One shared server-owned status/list/filter/count model | Campaign #65 and Core #263 | DataGrid contract/build tests plus exact shortcut/query/filter/count and large-result behavior | Complete 2026-07-22 (`e6062fe`, `cece71d`, `aa4ec66`, `4eb651c`) |
 | 6 | Campaign operator recovery | A durable campaign/version queue page exposes historical work, exact non-overlapping state counts, persisted mode, permission-safe controls, server-paged job evidence, bounded refresh and active-state recovery | Fixed-position actions, disabled explanations, leave/return state, version-scoped retry/queue/reconcile and explicit campaign-wide pause/resume/cancel | Campaign #78 | Queue model/structure, historical-version, permission, paging, recovery-control, stale-response and delta tests | Complete 2026-07-22 (`21f3014`, `99d44ee`, `735e874`) |
 | 7 | Campaign aggregate reports | A separate aggregate-reader projection and UI expose only policy-suppressed business totals with a stable status domain | Explicit denominator and exclusions, deployment floor plus tenant-strengthened small-cell threshold, complementary and overlapping-cell suppression, no detail/export/diagnostics | Campaign #80 | Aggregate query, cross-metric suppression, route/role/ACL, stable filter and UI structure tests | Complete 2026-07-22 (`06125cc`, `fc36aee`, `8ee87b7`, `ac3329c`, `1225802`) |
 | 8 | Campaign excluded outcomes | Excluded build rows become explicit skipped transport outcomes and remain protected from queue/cancel/retry ambiguity | One durable source-to-job-to-report meaning with guarded historical normalization | Campaign #66 | Builder/persistence, migration, query/count, queue-control and report-explanation tests | Complete 2026-07-22 (`7229fb8`) |
 | 9 | Guided first campaign | Existing wizard routes and ordinary workspace overlap | Task-oriented entry that hands off clearly to normal editing/review | Campaign #35 | First-run flow, resume/back, validation, optional modules, no implicit send | P1 after core pilot patterns stabilize |
-| 10 | Prove/extract generic primitives | Core already exports many primitives; Campaign composition still unreviewed | Extract only contracts with a second consumer or clear platform ownership | Core #225 plus bounded follow-ups | Core behavior/accessibility tests and module-permutation tests | After Campaign proof |
-| 11 | Configured-system pattern help | Docs route and classification exist | Role/config-aware pattern and route/field/blocker help | Docs #15 | Topic grouping, audience filtering, stable links/anchors | P1 after initial pattern IDs stabilize |
-| 12 | Admin/configuration family | Phase inventory and connector primitives exist in the ledger | Apply the pattern to files, mail, policy, retention, packages, modules, API keys, settings | Core #225 and module children | Per-surface state/accessibility/consequence evidence | Parallel where independent of Campaign shared decisions |
-| 13 | Remaining direct routes | Routes are contributed; most are unreviewed | Per-module bounded audit and migration plan | New module issues derived from this inventory | Applicable definition-of-done gates | P2 after Campaign, not a bulk rewrite |
+| 10 | Prove/extract generic primitives | Shared consequence, focus, help, blocker, unsaved-change, confirmation, connection-tree and effective-policy contracts now have Core and multiple module consumers | Keep Core behavior-only and leave domain composition in owning modules | Core #225 plus bounded follow-ups | Core behavior/accessibility tests and module-permutation tests | Complete 2026-08-03 (`fa32cca`; Files `d8ae506`; Mail `7844d9c`) |
+| 11 | Configured-system pattern help | Role/config-aware workflow, reference, pattern, and system topics are projected by Docs; shared route, field, blocker, and action links resolve to configured Docs or the hosted fallback | Stable configured-system guidance without feature-to-Docs imports | Docs #15 | Docs suite, shared component tests, Campaign review tests, 46 module permutations, full-product bundle budget | Complete 2026-08-03 (Docs `abe2f78`; Core `b823a22`; Campaign `d635f3a`) |
+| 12 | Admin/configuration family | Core host/settings/credential/retention contracts, shared primitives, module lifecycle, Files, Mail, Policy, Access, Admin, Tenancy, Views, and Organizations are integrated and verified | Continue the same consequence/provenance grammar only through bounded module-owned migrations | Core #225 and module children | Per-surface state/accessibility/consequence evidence | Core #225 complete `fa32cca`; Access `1409dbf`; Files `d8ae506`; Mail `7844d9c`; Policy `f964ed7`; Admin `d428f33`; Tenancy `e76fe16`; Views `c125f33`; Organizations `97acfcb` |
+| 13 | Remaining module surfaces | 33 bounded module-owned issues cover every WebUI contributor not already tracked by Campaign #74 or completed Docs #15 | Per-module audit and migration, ordered by user task and consequence rather than a bulk rewrite | Issues linked in the direct-route and composed-surface sections | Module-focused tests, manifest shapes, contextual Docs, and applicable definition-of-done gates | Scheduling `c17cbda`, Audit `6d3fcc1`, Access `1409dbf`, Files `d8ae506`, Mail `7844d9c`, Policy `f964ed7`, Admin `d428f33`, Tenancy `e76fe16`, Views `c125f33`, Organizations `97acfcb`, Postbox `a97eb3b`, IDM `d864317`, Committee `e64af30`, Approvals `24e9559`, Forms Runtime `07dd35b`, Forms `e505536`, Voting `2625990`, Distribution Lists `6cdd804`, Templates `72fafa2`, Addresses `f9a7185`, Datasources `6406ce7`, Dataflow `109ddcd`, Dashboard `da3947f`, Cases `43b4cc8`, Calendar `d7fd944`, Ops `2b32643`, Notifications `ad6a31f`, and Risk Compliance `24d80a6` complete |
 | 14 | Manifest/runtime alignment | Several executable routes are absent from manifest metadata | Declared alignment or explicit validated exception | Core contract issue to create | Automated manifest/module route check and configured Docs verification | Discovery follow-up |
 
 Workflow remains outside this rollout matrix because it has its own runtime and

docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md

@@ -44,6 +44,10 @@ least one check. The ledger verifies its hash chain before evidence is trusted.
 This is a platform contract, not an assertion that every existing module
 operation has adopted it. Module operations with external or multi-resource
 effects must be migrated to the ledger before claiming these guarantees.
+The owning-module inventory and adoption state are maintained in
+[Recovery Ledger Adoption](RECOVERY_LEDGER_ADOPTION.md); CI validates the
+machine-readable inventory so newly identified boundaries cannot disappear from
+the backlog silently.
 
 ## Deployment Journal
 
@@ -98,11 +102,15 @@ old code may not understand the new schema. Recovery then means one of:
 3. restore a separately verified, coordinated database/object/key backup and
    then deploy the matching release.
 
-The deployment tool does not create or validate that database backup. A
-`backup-required` annotation on the Kubernetes migration Job is an operator
-gate, not backup evidence. Production automation must provide a backup hook or
-external backup controller whose artifact, timestamp, scope, encryption key,
-and restore test can be referenced from the recovery record.
+The deployment tool does not create that backup. It does verify an externally
+produced, signed evidence contract covering PostgreSQL, objects, protected
+configuration, and key custody at one recovery point plus an isolated restore
+drill. A self-hosted release change cannot reach the migration command or be
+exported as a Kubernetes migration Job until fresh evidence bound to the
+previous immutable release has been adopted. Compose verifies it again after
+runtime quiescing. See
+[Backup And Restore Evidence](BACKUP_AND_RESTORE_EVIDENCE.md) for the contract,
+provider runbooks, RPO/RTO ownership, retention, and disposal rules.
 
 ## Scaled Nodes
 

docs/RECOVERY_LEDGER_ADOPTION.md

@@ -0,0 +1,89 @@
+# Recovery Ledger Adoption
+
+The Core recovery ledger is a platform primitive, not automatic protection for
+module-owned effects. The canonical, machine-checked inventory is
+[`recovery-operation-inventory.json`](recovery-operation-inventory.json).
+
+## Classification Rules
+
+- Use `atomic` only when every mutation commits in one database transaction and
+  no external effect occurs.
+- Use `compensation` when every completed effect has a bounded, verifiable
+  inverse action. A best-effort delete is not proof of compensation.
+- Use `snapshot_restore` only with fresh, signed backup evidence that covers all
+  affected state services at one recovery point.
+- Use `forward_recovery` for provider acceptance, queue publication, cursor
+  advancement, and other effects that may be resumable but cannot safely be
+  undone.
+- Use `irreversible` for approved purge or destruction where no automated
+  recovery is claimed.
+
+One feature may cross more than one boundary. Module installation is
+compensatable before schema migration, forward-only after migration starts, and
+snapshot-restorable for an approved destructive retirement. Mail submission is
+forward recovery because losing the response after provider acceptance must not
+cause an automatic resend.
+
+## Adoption Order
+
+1. Campaign build is the reference implementation for a database plus object
+   storage operation. Its operation reserves a build-specific object prefix,
+   persists request and precondition evidence before writes, records the final
+   object manifest, and verifies database/object state before success.
+2. Campaign delivery and Mail provider effects adopt outcome-unknown semantics
+   without weakening their existing provider-specific idempotency records.
+3. Files applies the same contract to uploads, purge, integrity reconciliation,
+   and writable connector synchronization.
+4. Connectors, Dataflow, and Workflow Engine consume the contract at their
+   registry/capability boundaries so optional providers remain optional.
+5. Core module lifecycle uses the ledger in addition to, not instead of, signed
+   deployment and backup evidence.
+
+Every fenced operation uses a process incarnation and distributed lease. A
+stale process cannot append a checkpoint or report success. An expired operation
+is claimed for recovery through an explicit takeover that preserves the prior
+fence in the checkpoint chain; it is never resumed as a normal retry.
+
+Connectors read-only sanctions and feed acquisitions are adopted: source
+revision/cursor and dry-run evidence are recorded before provider I/O, while
+the immutable snapshot and terminal checkpoint commit atomically. The generic
+external-mutation contract is conformance-tested but remains `planned` until a
+production connector actually publishes, updates, or deletes provider state.
+
+Dataflow runs are adopted. Database-only execution uses one atomic terminal
+commit for the run projection and recovery checkpoint. Output publication uses
+forward recovery: source and output digests are checkpointed before dispatch,
+a conclusive provider result commits with the run projection, and an expired
+or failed attempt after dispatch becomes `outcome_unknown`. A stale attempt may
+be retried only when its durable boundary proves dispatch had not started.
+
+Workflow Engine is adopted at both declared boundaries. Instance workers,
+trigger deliveries, and timer resumptions use process-bound distributed fences.
+Every module-action invocation records the pinned definition, input, preview,
+authority, provider-idempotency, and action-contract hashes before dispatch.
+Conclusive results commit with the Workflow projection. A lost acknowledgement,
+invalid result, or unannounced non-atomic effect becomes `outcome_unknown` and
+cannot be retried until evidence confirms either that the effect occurred or is
+absent. Linked Dataflow uncertainty blocks the Workflow without duplicating
+Dataflow's recovery authority.
+
+Core module lifecycle is adopted at four boundaries. Installer recovery is
+prepared before snapshots so a full database restore preserves the attempted
+operation. Pre-migration package changes use compensation, migrated changes use
+forward recovery, destructive retirement requires a hashed and restore-checked
+snapshot, and live graph changes restore the prior registry when no migration
+ran. A deployment-wide database fence serializes these effects; any unresolved
+predecessor blocks a differently keyed retry until explicit reconciliation.
+Supervised installs become successful only after restart and health evidence is
+recorded.
+
+## Operator Contract
+
+Ops lists non-terminal and manual-intervention operations. Operators must verify
+the checkpoint chain before trusting evidence, distinguish `outcome_unknown`
+from rejection, and use the owning module's documented reconciliation action.
+No evidence payload may contain credentials or resolved secrets.
+
+The parent adoption issue remains open until all inventory rows are adopted and
+the module matrix proves crash, retry, stale-fence, tamper, and optional-module
+behavior for each consequential path.

docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md

@@ -190,14 +190,16 @@ access, node visibility, drain controls, migration serialization, and scheduler
 fencing. It does not by itself provide:
 
 - a highly available PostgreSQL, Redis, or object-store deployment;
-- automatic PostgreSQL backup, point-in-time recovery, or restore verification;
+- automatic PostgreSQL/object backup creation or point-in-time recovery;
 - autoscaling policy;
 - central logs, metrics, traces, or alert routing;
 - certificate portability between independently managed ingress providers;
 - automatic reconciliation of every possible module side effect;
 - a service-level availability guarantee.
 
-Those are deployment and module-adoption requirements. Before claiming high
+The deployer verifies and gates migrations on signed coordinated backup and
+isolated-restore evidence, but backup capture and restoration remain owned by
+the selected state-service providers. Before claiming high
 availability, drill replica loss, rolling replacement, session continuity, job
 redelivery, scheduler failover, migration exclusion, object-store outage, and a
 coordinated database/object/key restore. Recovery rules and evidence are

docs/backup-evidence-keyring.schema.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://govoplan.add-ideas.de/schemas/backup-evidence-keyring-v1.json",
+  "title": "GovOPlaN backup evidence trust keyring",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "purpose", "keys"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "purpose": { "const": "govoplan-backup-evidence" },
+    "keys": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 64,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "key_id",
+          "algorithm",
+          "status",
+          "public_key_pem",
+          "not_before",
+          "expires_at"
+        ],
+        "properties": {
+          "key_id": { "type": "string", "minLength": 1, "maxLength": 128 },
+          "algorithm": { "const": "ed25519" },
+          "status": { "enum": ["active", "retired", "revoked"] },
+          "public_key_pem": { "type": "string", "minLength": 1, "maxLength": 8192 },
+          "not_before": { "type": "string", "format": "date-time" },
+          "expires_at": { "type": "string", "format": "date-time" }
+        }
+      }
+    }
+  }
+}

docs/backup-evidence.schema.json

@@ -0,0 +1,255 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://govoplan.add-ideas.de/schemas/backup-evidence-v1.json",
+  "title": "GovOPlaN coordinated backup and restore evidence",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "evidence_id",
+    "installation_id",
+    "deployment_subject",
+    "release",
+    "recovery_point",
+    "components",
+    "restore_drill",
+    "issued_at",
+    "expires_at",
+    "revoked",
+    "signatures"
+  ],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "evidence_id": { "$ref": "#/$defs/token" },
+    "installation_id": { "$ref": "#/$defs/token" },
+    "deployment_subject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["profile", "topology", "subject_ref"],
+      "properties": {
+        "profile": { "enum": ["evaluation", "self-hosted"] },
+        "topology": { "$ref": "#/$defs/token" },
+        "subject_ref": { "$ref": "#/$defs/reference" }
+      }
+    },
+    "release": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "channel",
+        "version",
+        "manifest_sha256",
+        "composition_sha256",
+        "api_image",
+        "web_image"
+      ],
+      "properties": {
+        "channel": { "$ref": "#/$defs/token" },
+        "version": { "$ref": "#/$defs/token" },
+        "manifest_sha256": { "$ref": "#/$defs/sha256" },
+        "composition_sha256": { "$ref": "#/$defs/sha256" },
+        "api_image": { "$ref": "#/$defs/digest_image" },
+        "web_image": { "$ref": "#/$defs/digest_image" }
+      }
+    },
+    "recovery_point": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "captured_at", "consistency", "rpo_seconds", "write_fence"],
+      "properties": {
+        "id": { "$ref": "#/$defs/token" },
+        "captured_at": { "type": "string", "format": "date-time" },
+        "consistency": {
+          "enum": ["provider-atomic", "application-quiesced", "transaction-consistent"]
+        },
+        "rpo_seconds": { "$ref": "#/$defs/duration" },
+        "write_fence": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["mode", "token_sha256", "established_at"],
+          "properties": {
+            "mode": {
+              "enum": ["provider-snapshot", "application-quiesce", "transaction-boundary"]
+            },
+            "token_sha256": { "$ref": "#/$defs/sha256" },
+            "established_at": { "type": "string", "format": "date-time" }
+          }
+        }
+      }
+    },
+    "components": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["database", "objects", "configuration", "key_custody"],
+      "properties": {
+        "database": { "$ref": "#/$defs/database" },
+        "objects": { "$ref": "#/$defs/objects" },
+        "configuration": { "$ref": "#/$defs/configuration" },
+        "key_custody": { "$ref": "#/$defs/key_custody" }
+      }
+    },
+    "restore_drill": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "drill_id",
+        "recovery_point_id",
+        "started_at",
+        "completed_at",
+        "isolated_target_ref",
+        "release_manifest_sha256",
+        "migration_heads_sha256",
+        "representative_object_manifest_sha256",
+        "database_verified",
+        "objects_verified",
+        "configuration_verified",
+        "key_custody_verified",
+        "semantic_checks",
+        "measured_rpo_seconds",
+        "measured_rto_seconds",
+        "evidence_ref"
+      ],
+      "properties": {
+        "drill_id": { "$ref": "#/$defs/token" },
+        "recovery_point_id": { "$ref": "#/$defs/token" },
+        "started_at": { "type": "string", "format": "date-time" },
+        "completed_at": { "type": "string", "format": "date-time" },
+        "isolated_target_ref": { "$ref": "#/$defs/reference" },
+        "release_manifest_sha256": { "$ref": "#/$defs/sha256" },
+        "migration_heads_sha256": { "$ref": "#/$defs/sha256" },
+        "representative_object_manifest_sha256": { "$ref": "#/$defs/sha256" },
+        "database_verified": { "const": true },
+        "objects_verified": { "const": true },
+        "configuration_verified": { "const": true },
+        "key_custody_verified": { "const": true },
+        "semantic_checks": {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 128,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["id", "status", "evidence_ref"],
+            "properties": {
+              "id": { "$ref": "#/$defs/token" },
+              "status": { "const": "passed" },
+              "evidence_ref": { "$ref": "#/$defs/reference" }
+            }
+          }
+        },
+        "measured_rpo_seconds": { "$ref": "#/$defs/duration" },
+        "measured_rto_seconds": { "$ref": "#/$defs/duration" },
+        "evidence_ref": { "$ref": "#/$defs/reference" }
+      }
+    },
+    "issued_at": { "type": "string", "format": "date-time" },
+    "expires_at": { "type": "string", "format": "date-time" },
+    "revoked": { "const": false },
+    "signatures": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 16,
+      "items": { "$ref": "#/$defs/signature" }
+    }
+  },
+  "$defs": {
+    "token": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128,
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"
+    },
+    "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+    "digest_image": {
+      "type": "string",
+      "maxLength": 300,
+      "pattern": "^[^@\\s]+@sha256:[0-9a-f]{64}$"
+    },
+    "reference": {
+      "type": "string",
+      "minLength": 3,
+      "maxLength": 2048,
+      "pattern": "^[A-Za-z][A-Za-z0-9+.-]*:[^\\s]+$"
+    },
+    "duration": { "type": "integer", "minimum": 0, "maximum": 2592000 },
+    "protected_key": {
+      "type": "object",
+      "properties": {
+        "protected": { "const": true },
+        "encryption_key_ref": { "$ref": "#/$defs/reference" },
+        "captured_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "database": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "provider", "artifact_ref", "artifact_sha256", "snapshot_id", "lsn",
+        "protected", "encryption_key_ref", "captured_at"
+      ],
+      "properties": {
+        "provider": { "$ref": "#/$defs/token" },
+        "artifact_ref": { "$ref": "#/$defs/reference" },
+        "artifact_sha256": { "$ref": "#/$defs/sha256" },
+        "snapshot_id": { "$ref": "#/$defs/token" },
+        "lsn": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "protected": { "const": true },
+        "encryption_key_ref": { "$ref": "#/$defs/reference" },
+        "captured_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "objects": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "provider", "artifact_ref", "manifest_sha256", "version_id",
+        "object_count", "total_bytes", "protected", "encryption_key_ref", "captured_at"
+      ],
+      "properties": {
+        "provider": { "$ref": "#/$defs/token" },
+        "artifact_ref": { "$ref": "#/$defs/reference" },
+        "manifest_sha256": { "$ref": "#/$defs/sha256" },
+        "version_id": { "$ref": "#/$defs/token" },
+        "object_count": { "type": "integer", "minimum": 0 },
+        "total_bytes": { "type": "integer", "minimum": 0 },
+        "protected": { "const": true },
+        "encryption_key_ref": { "$ref": "#/$defs/reference" },
+        "captured_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["artifact_ref", "sha256", "protected", "encryption_key_ref", "captured_at"],
+      "properties": {
+        "artifact_ref": { "$ref": "#/$defs/reference" },
+        "sha256": { "$ref": "#/$defs/sha256" },
+        "protected": { "const": true },
+        "encryption_key_ref": { "$ref": "#/$defs/reference" },
+        "captured_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "key_custody": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"],
+      "properties": {
+        "provider": { "$ref": "#/$defs/token" },
+        "keyset_ref": { "$ref": "#/$defs/reference" },
+        "keyset_version": { "$ref": "#/$defs/token" },
+        "recoverable": { "const": true },
+        "captured_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "signature": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["key_id", "algorithm", "value"],
+      "properties": {
+        "key_id": { "$ref": "#/$defs/token" },
+        "algorithm": { "const": "ed25519" },
+        "value": { "type": "string", "minLength": 1, "maxLength": 256 }
+      }
+    }
+  }
+}

docs/recovery-operation-inventory.json

@@ -0,0 +1,195 @@
+{
+  "schema_version": 1,
+  "parent_issue": "https://git.add-ideas.de/GovOPlaN/govoplan/issues/36",
+  "operations": [
+    {
+      "id": "campaign.build.publish-artifacts",
+      "repository": "govoplan-campaign",
+      "resources": ["postgresql", "object-storage", "templates-capability", "files-capability"],
+      "mode": "compensation",
+      "fenced": true,
+      "adoption": "reference-implementation",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
+    },
+    {
+      "id": "campaign.delivery.external-channels",
+      "repository": "govoplan-campaign",
+      "resources": ["postgresql", "queue", "smtp", "imap", "postbox", "print-provider"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
+    },
+    {
+      "id": "campaign.retention.generated-artifacts",
+      "repository": "govoplan-campaign",
+      "resources": ["postgresql", "object-storage"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
+    },
+    {
+      "id": "files.upload.finalize",
+      "repository": "govoplan-files",
+      "resources": ["postgresql", "object-storage", "filesystem-staging"],
+      "mode": "compensation",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
+    },
+    {
+      "id": "files.retention.purge",
+      "repository": "govoplan-files",
+      "resources": ["postgresql", "object-storage", "encryption-key-custody"],
+      "mode": "irreversible",
+      "fenced": true,
+      "adoption": "planned",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
+    },
+    {
+      "id": "files.integrity.reconcile",
+      "repository": "govoplan-files",
+      "resources": ["postgresql", "object-storage"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
+    },
+    {
+      "id": "files.connector.write-sync",
+      "repository": "govoplan-files",
+      "resources": ["postgresql", "object-storage", "external-connector"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "planned",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
+    },
+    {
+      "id": "mail.outbox.smtp-submit",
+      "repository": "govoplan-mail",
+      "resources": ["postgresql", "queue", "smtp"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
+    },
+    {
+      "id": "mail.sent.imap-append",
+      "repository": "govoplan-mail",
+      "resources": ["postgresql", "imap"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
+    },
+    {
+      "id": "mail.mailbox.imap-mutate",
+      "repository": "govoplan-mail",
+      "resources": ["postgresql", "imap"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "planned",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
+    },
+    {
+      "id": "mail.mailbox.sync-cursor",
+      "repository": "govoplan-mail",
+      "resources": ["postgresql", "imap"],
+      "mode": "atomic",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
+    },
+    {
+      "id": "connectors.sync.read-snapshot",
+      "repository": "govoplan-connectors",
+      "resources": ["postgresql", "external-provider"],
+      "mode": "atomic",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15"
+    },
+    {
+      "id": "connectors.sync.external-mutation",
+      "repository": "govoplan-connectors",
+      "resources": ["postgresql", "queue", "external-provider"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "planned",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15"
+    },
+    {
+      "id": "dataflow.run.database-only",
+      "repository": "govoplan-dataflow",
+      "resources": ["postgresql"],
+      "mode": "atomic",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19"
+    },
+    {
+      "id": "dataflow.run.publish-output",
+      "repository": "govoplan-dataflow",
+      "resources": ["postgresql", "queue", "object-storage", "external-sink"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19"
+    },
+    {
+      "id": "workflow-engine.instance.state-transition",
+      "repository": "govoplan-workflow-engine",
+      "resources": ["postgresql"],
+      "mode": "atomic",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1"
+    },
+    {
+      "id": "workflow-engine.activity.external-effect",
+      "repository": "govoplan-workflow-engine",
+      "resources": ["postgresql", "queue", "module-capability", "external-provider"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1"
+    },
+    {
+      "id": "core.module-lifecycle.pre-migration",
+      "repository": "govoplan-core",
+      "resources": ["postgresql", "package-environment", "webui-bundle", "filesystem"],
+      "mode": "compensation",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
+    },
+    {
+      "id": "core.module-lifecycle.post-migration",
+      "repository": "govoplan-core",
+      "resources": ["postgresql", "package-environment", "webui-bundle", "runtime-nodes"],
+      "mode": "forward_recovery",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
+    },
+    {
+      "id": "core.module-retirement.destroy-data",
+      "repository": "govoplan-core",
+      "resources": ["postgresql", "object-storage", "package-environment"],
+      "mode": "snapshot_restore",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
+    },
+    {
+      "id": "core.module-runtime.apply-graph",
+      "repository": "govoplan-core",
+      "resources": ["postgresql", "runtime-nodes", "module-registry"],
+      "mode": "compensation",
+      "fenced": true,
+      "adoption": "adopted",
+      "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
+    }
+  ]
+}

tests/test_backup_evidence.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+import base64
+from contextlib import redirect_stderr, redirect_stdout
+from datetime import UTC, datetime, timedelta
+import hashlib
+import io
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+
+META_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(META_ROOT / "tools" / "deployment"))
+
+from govoplan_deploy.backup_evidence import verify_backup_evidence  # noqa: E402
+from govoplan_deploy.bundle import (  # noqa: E402
+    atomic_write,
+    bundle_paths,
+    canonical_json,
+    read_env,
+)
+from govoplan_deploy.cli import main as deploy_main  # noqa: E402
+from govoplan_deploy.distribution import (  # noqa: E402
+    DistributionError,
+    canonical_signed_payload,
+    canonical_json as canonical_distribution_json,
+)
+from govoplan_deploy.model import default_spec, parse_spec  # noqa: E402
+from govoplan_deploy.planning import (  # noqa: E402
+    release_change_requires_backup,
+    verify_stored_backup_evidence,
+)
+
+
+class BackupEvidenceTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.now = datetime(2026, 8, 3, 12, tzinfo=UTC)
+        self.private = Ed25519PrivateKey.generate()
+        public = (
+            self.private.public_key()
+            .public_bytes(
+                serialization.Encoding.PEM,
+                serialization.PublicFormat.SubjectPublicKeyInfo,
+            )
+            .decode("ascii")
+        )
+        self.keyring = {
+            "schema_version": "1",
+            "purpose": "govoplan-backup-evidence",
+            "keys": [
+                {
+                    "key_id": "backup-controller-1",
+                    "algorithm": "ed25519",
+                    "status": "active",
+                    "public_key_pem": public,
+                    "not_before": (self.now - timedelta(days=1)).isoformat(),
+                    "expires_at": (self.now + timedelta(days=365)).isoformat(),
+                }
+            ],
+        }
+        self.release = {
+            "channel": "stable",
+            "version": "1.2.3",
+            "manifest_sha256": "a" * 64,
+            "composition_sha256": "b" * 64,
+            "api_image": "registry.example/api@sha256:" + "c" * 64,
+            "web_image": "registry.example/web@sha256:" + "d" * 64,
+        }
+
+    def test_verifies_coordinated_restore_drill_and_release_binding(self) -> None:
+        summary = verify_backup_evidence(
+            self._evidence(),
+            self.keyring,
+            installation_id="govoplan-test",
+            profile="self-hosted",
+            release=self.release,
+            now=self.now,
+        )
+
+        self.assertEqual("recovery-1", summary["recovery_point_id"])
+        self.assertEqual("restore-1", summary["restore_drill_id"])
+        self.assertEqual("backup-controller-1", summary["signature_key_id"])
+
+    def test_tampering_staleness_and_partial_restore_fail_closed(self) -> None:
+        tampered = self._evidence()
+        tampered["components"]["objects"]["object_count"] = 999
+        with self.assertRaisesRegex(DistributionError, "signature verification"):
+            verify_backup_evidence(
+                tampered,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+        stale = self._evidence(captured=self.now - timedelta(days=2))
+        with self.assertRaisesRegex(DistributionError, "stale"):
+            verify_backup_evidence(
+                stale,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+        partial = self._evidence()
+        partial["restore_drill"]["objects_verified"] = False
+        partial["signatures"] = [self._signature(partial)]
+        with self.assertRaisesRegex(DistributionError, "objects_verified"):
+            verify_backup_evidence(
+                partial,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+    def test_wrong_release_key_purpose_and_component_skew_fail_closed(self) -> None:
+        wrong_release = dict(self.release)
+        wrong_release["version"] = "1.2.4"
+        with self.assertRaisesRegex(DistributionError, "release field"):
+            verify_backup_evidence(
+                self._evidence(),
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=wrong_release,
+                now=self.now,
+            )
+
+        wrong_keyring = dict(self.keyring)
+        wrong_keyring["purpose"] = "govoplan-runtime-distribution"
+        with self.assertRaisesRegex(DistributionError, "wrong purpose"):
+            verify_backup_evidence(
+                self._evidence(),
+                wrong_keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+        skewed = self._evidence()
+        skewed["components"]["database"]["captured_at"] = (
+            self.now - timedelta(hours=1)
+        ).isoformat()
+        skewed["signatures"] = [self._signature(skewed)]
+        with self.assertRaisesRegex(DistributionError, "one recovery point"):
+            verify_backup_evidence(
+                skewed,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+        false_rto = self._evidence()
+        false_rto["restore_drill"]["measured_rto_seconds"] = 1
+        false_rto["signatures"] = [self._signature(false_rto)]
+        with self.assertRaisesRegex(DistributionError, "RTO"):
+            verify_backup_evidence(
+                false_rto,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+                now=self.now,
+            )
+
+    def test_provider_signing_tool_emits_canonical_verified_evidence(self) -> None:
+        self.now = datetime.now(UTC)
+        self.keyring["keys"][0]["not_before"] = (
+            self.now - timedelta(days=1)
+        ).isoformat()
+        self.keyring["keys"][0]["expires_at"] = (
+            self.now + timedelta(days=365)
+        ).isoformat()
+        evidence = self._evidence()
+        evidence["signatures"] = []
+
+        with tempfile.TemporaryDirectory(prefix="govoplan-backup-signer-") as value:
+            root = Path(value)
+            source = root / "unsigned.json"
+            output = root / "signed.json"
+            keyring = root / "keyring.json"
+            private_key = root / "private.pem"
+            atomic_write(source, canonical_json(evidence), mode=0o600)
+            atomic_write(keyring, canonical_json(self.keyring), mode=0o600)
+            atomic_write(
+                private_key,
+                self.private.private_bytes(
+                    serialization.Encoding.PEM,
+                    serialization.PrivateFormat.PKCS8,
+                    serialization.NoEncryption(),
+                ),
+                mode=0o600,
+            )
+
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    str(META_ROOT / "tools/deployment/sign-backup-evidence.py"),
+                    "--input",
+                    str(source),
+                    "--output",
+                    str(output),
+                    "--trusted-keyring",
+                    str(keyring),
+                    "--signing-key",
+                    f"backup-controller-1={private_key}",
+                ],
+                cwd=META_ROOT,
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+
+            self.assertEqual(0, result.returncode, result.stderr)
+            encoded = output.read_bytes()
+            signed = json.loads(encoded)
+            self.assertEqual(canonical_distribution_json(signed), encoded)
+            summary = verify_backup_evidence(
+                signed,
+                self.keyring,
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                release=self.release,
+            )
+            self.assertEqual("backup-controller-1", summary["signature_key_id"])
+
+    def test_cli_adoption_gates_the_next_release_against_previous_receipt(self) -> None:
+        self.now = datetime.now(UTC)
+        self.keyring["keys"][0]["not_before"] = (
+            self.now - timedelta(days=1)
+        ).isoformat()
+        self.keyring["keys"][0]["expires_at"] = (
+            self.now + timedelta(days=365)
+        ).isoformat()
+        evidence = self._evidence()
+        encoded_evidence = canonical_distribution_json(evidence)
+        encoded_keyring = canonical_distribution_json(self.keyring)
+
+        with tempfile.TemporaryDirectory(prefix="govoplan-backup-evidence-") as value:
+            paths = bundle_paths(Path(value))
+            paths.root.chmod(0o700)
+            raw = default_spec(
+                installation_id="govoplan-test",
+                profile="self-hosted",
+                public_url="https://govoplan.example.test",
+                ingress_mode="existing-proxy",
+                trusted_proxy_cidrs=("127.0.0.1/32",),
+            ).to_dict()
+            raw["release"] = {
+                **raw["release"],
+                **self.release,
+            }
+            current = parse_spec(raw)
+            atomic_write(paths.spec, canonical_json(current.to_dict()), mode=0o600)
+            source_evidence = paths.root / "source-backup.json"
+            source_keyring = paths.root / "source-keyring.json"
+            atomic_write(source_evidence, encoded_evidence, mode=0o600)
+            atomic_write(source_keyring, encoded_keyring, mode=0o600)
+
+            output = io.StringIO()
+            with redirect_stdout(output), redirect_stderr(output):
+                result = deploy_main(
+                    [
+                        "verify-backup",
+                        "--directory",
+                        str(paths.root),
+                        "--evidence",
+                        str(source_evidence),
+                        "--evidence-sha256",
+                        hashlib.sha256(encoded_evidence).hexdigest(),
+                        "--trusted-keyring",
+                        str(source_keyring),
+                        "--adopt",
+                    ]
+                )
+            self.assertEqual(0, result, output.getvalue())
+            runtime_environment = read_env(paths.env)
+            self.assertEqual(
+                "verified",
+                runtime_environment["GOVOPLAN_BACKUP_EVIDENCE_STATE"],
+            )
+            self.assertEqual(
+                "recovery-1",
+                runtime_environment["GOVOPLAN_BACKUP_RECOVERY_POINT_ID"],
+            )
+            self.assertNotIn("snapshot:postgres", str(runtime_environment))
+            self.assertNotIn("urn:kms", str(runtime_environment))
+
+            receipt = {
+                "installation_id": current.installation_id,
+                "profile": current.profile,
+                "release": dict(self.release),
+            }
+            atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
+            target_raw = current.to_dict()
+            target_raw["release"]["version"] = "1.2.4"
+            target_raw["release"]["manifest_sha256"] = "9" * 64
+            target = parse_spec(target_raw)
+
+            self.assertTrue(release_change_requires_backup(target, receipt))
+            summary = verify_stored_backup_evidence(
+                target,
+                paths,
+                receipt=receipt,
+            )
+            self.assertEqual("recovery-1", summary["recovery_point_id"])
+
+    def _evidence(self, *, captured: datetime | None = None) -> dict[str, object]:
+        captured = captured or self.now - timedelta(hours=2)
+        started = captured + timedelta(minutes=15)
+        completed = captured + timedelta(minutes=30)
+        issued = completed + timedelta(minutes=10)
+        artifact_time = captured.isoformat()
+        payload: dict[str, object] = {
+            "schema_version": "1",
+            "evidence_id": "backup-1",
+            "installation_id": "govoplan-test",
+            "deployment_subject": {
+                "profile": "self-hosted",
+                "topology": "compose",
+                "subject_ref": "urn:govoplan:installation:govoplan-test",
+            },
+            "release": dict(self.release),
+            "recovery_point": {
+                "id": "recovery-1",
+                "captured_at": captured.isoformat(),
+                "consistency": "application-quiesced",
+                "rpo_seconds": 300,
+                "write_fence": {
+                    "mode": "application-quiesce",
+                    "token_sha256": "e" * 64,
+                    "established_at": captured.isoformat(),
+                },
+            },
+            "components": {
+                "database": {
+                    "provider": "postgres",
+                    "artifact_ref": "snapshot:postgres:backup-1",
+                    "artifact_sha256": "1" * 64,
+                    "snapshot_id": "pg-snapshot-1",
+                    "lsn": "0/16B6C50",
+                    "protected": True,
+                    "encryption_key_ref": "urn:kms:key:database-backup",
+                    "captured_at": artifact_time,
+                },
+                "objects": {
+                    "provider": "s3",
+                    "artifact_ref": "s3://backup/govoplan-test/recovery-1",
+                    "manifest_sha256": "2" * 64,
+                    "version_id": "object-snapshot-1",
+                    "object_count": 4,
+                    "total_bytes": 1024,
+                    "protected": True,
+                    "encryption_key_ref": "urn:kms:key:object-backup",
+                    "captured_at": artifact_time,
+                },
+                "configuration": {
+                    "artifact_ref": "backup:configuration:recovery-1",
+                    "sha256": "3" * 64,
+                    "protected": True,
+                    "encryption_key_ref": "urn:kms:key:configuration-backup",
+                    "captured_at": artifact_time,
+                },
+                "key_custody": {
+                    "provider": "kms",
+                    "keyset_ref": "urn:kms:keyset:govoplan-test",
+                    "keyset_version": "version-4",
+                    "recoverable": True,
+                    "captured_at": artifact_time,
+                },
+            },
+            "restore_drill": {
+                "drill_id": "restore-1",
+                "recovery_point_id": "recovery-1",
+                "started_at": started.isoformat(),
+                "completed_at": completed.isoformat(),
+                "isolated_target_ref": "urn:govoplan:restore-target:restore-1",
+                "release_manifest_sha256": self.release["manifest_sha256"],
+                "migration_heads_sha256": "4" * 64,
+                "representative_object_manifest_sha256": "2" * 64,
+                "database_verified": True,
+                "objects_verified": True,
+                "configuration_verified": True,
+                "key_custody_verified": True,
+                "semantic_checks": [
+                    {
+                        "id": "institutional-journey",
+                        "status": "passed",
+                        "evidence_ref": "evidence:journey:institutional-1",
+                    }
+                ],
+                "measured_rpo_seconds": 120,
+                "measured_rto_seconds": 900,
+                "evidence_ref": "evidence:restore:restore-1",
+            },
+            "issued_at": issued.isoformat(),
+            "expires_at": (self.now + timedelta(days=7)).isoformat(),
+            "revoked": False,
+            "signatures": [],
+        }
+        payload["signatures"] = [self._signature(payload)]
+        return payload
+
+    def _signature(self, payload: dict[str, object]) -> dict[str, str]:
+        return {
+            "key_id": "backup-controller-1",
+            "algorithm": "ed25519",
+            "value": base64.b64encode(
+                self.private.sign(canonical_signed_payload(payload))
+            ).decode("ascii"),
+        }
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_deployment_installer.py

@@ -318,7 +318,16 @@ class DeploymentInstallerTests(unittest.TestCase):
             },
         )
 
-        manifest = render_kubernetes(spec, environment)
+        manifest = render_kubernetes(
+            spec,
+            environment,
+            backup_required=True,
+            backup_evidence={
+                "evidence_sha256": "c" * 64,
+                "recovery_point_id": "recovery-1",
+                "restore_drill_id": "drill-1",
+            },
+        )
         rendered = json.dumps(manifest, sort_keys=True)
         kinds = [item["kind"] for item in manifest["items"]]
         deployments = {
@@ -351,6 +360,18 @@ class DeploymentInstallerTests(unittest.TestCase):
             "forward-recovery",
             migration["metadata"]["annotations"]["govoplan.add-ideas.de/recovery-mode"],
         )
+        self.assertEqual(
+            "c" * 64,
+            migration["metadata"]["annotations"][
+                "govoplan.add-ideas.de/backup-evidence-sha256"
+            ],
+        )
+        self.assertEqual(
+            "recovery-1",
+            migration["metadata"]["annotations"][
+                "govoplan.add-ideas.de/recovery-point"
+            ],
+        )
         config = next(item for item in manifest["items"] if item["kind"] == "ConfigMap")
         self.assertEqual("shared", config["data"]["GOVOPLAN_STATE_PROFILE"])
         self.assertEqual("3", config["data"]["GOVOPLAN_EXPECTED_API_REPLICAS"])
@@ -530,6 +551,9 @@ class DeploymentInstallerTests(unittest.TestCase):
         )
         self.assertIn("caddy-data:/data", ingress["volumes"])
         self.assertIn("caddy-config:/config", ingress["volumes"])
+        self.assertEqual(["ALL"], ingress["cap_drop"])
+        self.assertEqual(["NET_BIND_SERVICE"], ingress["cap_add"])
+        self.assertEqual(["no-new-privileges:true"], ingress["security_opt"])
         self.assertIn("reverse_proxy load-balancer:8080", render_caddy_config(spec))
         self.assertNotIn("operator@example.test", json.dumps(compose))
 

tests/test_managed_ingress_drill.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+import json
+import subprocess
+import sys
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _load_module():
+    path = ROOT / "tools/checks/managed-ingress-drill.py"
+    spec = importlib.util.spec_from_file_location("managed_ingress_drill", path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+INGRESS = _load_module()
+
+
+class ManagedIngressDrillTests(unittest.TestCase):
+    def test_config_is_streamed_into_a_daemon_visible_volume(self) -> None:
+        completed = subprocess.CompletedProcess([], 0, "", "")
+        with patch.object(INGRESS, "_run", return_value=completed) as run:
+            INGRESS._write_volume_file(
+                image="registry.example/caddy@sha256:" + "1" * 64,
+                volume="config-volume",
+                filename="Caddyfile",
+                content=":8080 { respond /health 200 }\n",
+            )
+
+        argv = run.call_args.args[0]
+        self.assertIn("type=volume,src=config-volume,dst=/govoplan-config", argv)
+        self.assertIn("0:0", argv)
+        self.assertNotIn("type=bind", " ".join(argv))
+        self.assertEqual(
+            ":8080 { respond /health 200 }\n",
+            run.call_args.kwargs["input_text"],
+        )
+
+    def test_config_filename_cannot_escape_the_volume(self) -> None:
+        with self.assertRaisesRegex(ValueError, "invalid config filename"):
+            INGRESS._write_volume_file(
+                image="registry.example/caddy@sha256:" + "1" * 64,
+                volume="config-volume",
+                filename="../Caddyfile",
+                content="",
+            )
+
+    def test_drill_has_no_runner_local_bind_mounts(self) -> None:
+        source = (ROOT / "tools/checks/managed-ingress-drill.py").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertNotIn("type=bind", source)
+        self.assertIn('"--network-alias",\n                    "load-balancer"', source)
+        self.assertNotIn('"127.0.0.1::8080"', source)
+        self.assertIn("requested_http_port", source)
+        self.assertIn("requested_https_port", source)
+        self.assertIn('"--cap-add",\n                "NET_BIND_SERVICE"', source)
+
+    def test_published_port_reads_the_docker_mapping(self) -> None:
+        completed = subprocess.CompletedProcess(
+            [],
+            0,
+            json.dumps(
+                {
+                    "8443/tcp": [
+                        {"HostIp": "127.0.0.1", "HostPort": "49152"}
+                    ]
+                }
+            ),
+            "",
+        )
+        with patch.object(INGRESS, "_run", return_value=completed) as run:
+            port = INGRESS._published_port("ingress", 8443)
+
+        self.assertEqual(49152, port)
+        self.assertEqual(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                "{{json .HostConfig.PortBindings}}",
+                "ingress",
+            ],
+            run.call_args.args[0],
+        )
+
+    def test_published_port_rejects_non_loopback_binding(self) -> None:
+        completed = subprocess.CompletedProcess(
+            [],
+            0,
+            '{"8443/tcp":[{"HostIp":"0.0.0.0","HostPort":"49152"}]}',
+            "",
+        )
+        with patch.object(INGRESS, "_run", return_value=completed):
+            with self.assertRaisesRegex(RuntimeError, "loopback binding"):
+                INGRESS._published_port("ingress", 8443)
+
+    def test_probe_runs_as_a_network_sibling_from_a_digest_image(self) -> None:
+        completed = subprocess.CompletedProcess([], 0, "", "")
+        image = "registry.example/runtime-api@sha256:" + "1" * 64
+        with patch.object(INGRESS, "_run", return_value=completed) as run:
+            INGRESS._probe_ingress(
+                image=image,
+                network="deployment-network",
+                container="ingress",
+            )
+
+        argv = run.call_args.args[0]
+        self.assertEqual("docker", argv[0])
+        self.assertIn("deployment-network", argv)
+        self.assertIn(image, argv)
+        self.assertIn('(\"ingress\", port)', argv[-1])
+        self.assertIn("server_hostname=\"localhost\"", argv[-1])
+        self.assertNotIn("localhost:49152", argv[-1])
+
+    def test_probe_diagnostics_include_container_stderr(self) -> None:
+        probe_failure = subprocess.CalledProcessError(1, ["docker", "run"])
+        state = subprocess.CompletedProcess([], 0, '{"Running":false}', "")
+        logs = subprocess.CompletedProcess([], 0, "", "caddy startup failed")
+        with patch.object(
+            INGRESS,
+            "_run",
+            side_effect=[probe_failure, state, logs],
+        ), patch.object(INGRESS.sys, "stderr") as stderr:
+            with self.assertRaises(subprocess.CalledProcessError):
+                INGRESS._probe_ingress(
+                    image="registry.example/runtime-api@sha256:" + "1" * 64,
+                    network="deployment-network",
+                    container="ingress",
+                )
+
+        rendered = "".join(call.args[0] for call in stderr.write.call_args_list)
+        self.assertIn('"Running":false', rendered)
+        self.assertIn("caddy startup failed", rendered)
+
+    def test_standalone_workflow_is_dispatch_only_and_digest_bounded(self) -> None:
+        workflow = (
+            ROOT / ".gitea/workflows/runtime-ingress-drill.yml"
+        ).read_text(encoding="utf-8")
+
+        self.assertIn("workflow_dispatch:", workflow)
+        self.assertNotIn("\n  push:", workflow)
+        self.assertIn("--probe-image \"$PROBE_IMAGE\"", workflow)
+        self.assertIn("GOVOPLAN_REGISTRY_TOKEN", workflow)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_python_environment_sync.py

@@ -113,6 +113,60 @@ class PythonEnvironmentSyncTests(unittest.TestCase):
         self.assertEqual(plan.mode, "Selective Python environment repair")
         self.assertEqual(plan.commands[0][-2:], ("-e", str(project_root)))
 
+    def test_metadata_sync_also_repairs_unrelated_missing_distribution(self) -> None:
+        sync = load_sync_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-python-sync-") as directory:
+            root = Path(directory)
+            requirements = root / "requirements-dev.txt"
+            requirements.write_text("-e ./govoplan-changed\n-e ./govoplan-missing\n", encoding="utf-8")
+            for project in ("govoplan-changed", "govoplan-missing"):
+                project_root = root / project
+                project_root.mkdir()
+                (project_root / "pyproject.toml").write_text(
+                    f'[project]\nname = "{project}"\nversion = "0.1.10"\n',
+                    encoding="utf-8",
+                )
+
+            entries = sync.local_requirement_entries(requirements)
+            fingerprint = sync.build_fingerprint(
+                requirements=requirements,
+                python="/test/venv/bin/python",
+                local_requirements=entries,
+            )
+            requirements_digest = hashlib.sha256(requirements.read_bytes()).hexdigest()
+            previous = {
+                "version": sync.STAMP_VERSION,
+                "python": "/test/venv/bin/python",
+                "inputs": [
+                    {"path": str(requirements), "sha256": requirements_digest},
+                    {"path": entries[0].pyproject, "sha256": "stale"},
+                    {
+                        "path": entries[1].pyproject,
+                        "sha256": hashlib.sha256(Path(entries[1].pyproject).read_bytes()).hexdigest(),
+                    },
+                ],
+                "requirements_entries": [
+                    entry.as_dict() for entry in sync.parse_requirement_entries(requirements)
+                ],
+            }
+
+            plan = sync.build_install_plan(
+                previous=previous,
+                fingerprint=fingerprint,
+                requirements=requirements,
+                python="/test/venv/bin/python",
+                local_requirements=entries,
+                repair_requirements=(entries[1],),
+                force=False,
+            )
+
+        self.assertEqual(plan.mode, "Selective Python environment sync")
+        self.assertEqual(len(plan.commands), 1)
+        command = plan.commands[0]
+        self.assertEqual(command.count("-e"), 2)
+        self.assertIn(str(root / "govoplan-changed"), command)
+        self.assertIn(str(root / "govoplan-missing"), command)
+
     def test_declared_module_entry_points_are_part_of_environment_validation(self) -> None:
         sync = load_sync_module()
         with tempfile.TemporaryDirectory(prefix="govoplan-python-sync-") as directory:

tests/test_recovery_operation_inventory.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+ROOT = Path(__file__).resolve().parents[1]
+INVENTORY = ROOT / "docs" / "recovery-operation-inventory.json"
+MODES = {
+    "atomic",
+    "compensation",
+    "snapshot_restore",
+    "forward_recovery",
+    "irreversible",
+}
+ADOPTION_STATES = {"planned", "reference-implementation", "adopted"}
+REQUIRED_PREFIXES = {
+    "campaign.",
+    "files.",
+    "mail.",
+    "connectors.",
+    "dataflow.",
+    "workflow-engine.",
+    "core.module-lifecycle.",
+    "core.module-runtime.",
+}
+ATOMIC_EXTERNAL_READS = {
+    "connectors.sync.read-snapshot",
+    "mail.mailbox.sync-cursor",
+}
+
+
+def test_recovery_operation_inventory_is_complete_and_actionable() -> None:
+    payload = json.loads(INVENTORY.read_text(encoding="utf-8"))
+    assert payload["schema_version"] == 1
+    operations = payload["operations"]
+    ids = [item["id"] for item in operations]
+    assert len(ids) == len(set(ids))
+    assert all(any(item.startswith(prefix) for item in ids) for prefix in REQUIRED_PREFIXES)
+
+    for item in operations:
+        assert item["mode"] in MODES
+        assert item["adoption"] in ADOPTION_STATES
+        assert item["repository"].startswith("govoplan-")
+        assert item["resources"]
+        assert item["fenced"] is True
+        issue = urlparse(item["issue"])
+        assert issue.scheme == "https"
+        assert issue.netloc == "git.add-ideas.de"
+        assert issue.path.startswith(f"/GovOPlaN/{item['repository']}/issues/")
+
+
+def test_non_atomic_operations_do_not_claim_plain_database_rollback() -> None:
+    operations = json.loads(INVENTORY.read_text(encoding="utf-8"))["operations"]
+    for item in operations:
+        if item["mode"] == "atomic":
+            assert (
+                item["resources"] == ["postgresql"]
+                or item["id"] in ATOMIC_EXTERNAL_READS
+            )

tests/test_release_repository_tag.py

@@ -340,6 +340,7 @@ def add_scoped_workflow_manifest(repo: Path) -> None:
     (backend / "__init__.py").write_text("", encoding="utf-8")
     (backend / "manifest.py").write_text(
         """from govoplan_core.core.modules import DocumentationCondition, DocumentationTopic, ModuleManifest, PermissionDefinition
+from govoplan_core.core.provider_governance import declared_module_architecture
 
 
 def get_manifest():
@@ -368,6 +369,20 @@ def get_manifest():
                 conditions=(DocumentationCondition(required_scopes=("access:item:read",)),),
                 metadata={"kind": "workflow"},
             ),
+            DocumentationTopic(
+                id="access.admin.reference",
+                title="Administer access",
+                summary="Static administrator documentation for the release fixture.",
+                documentation_types=("admin",),
+                metadata={"kind": "reference"},
+            ),
+        ),
+        architecture=declared_module_architecture(
+            layer="institutional_foundation",
+            kind="foundation",
+            maturity="scaffold",
+            documentation_ref="pyproject.toml",
+            known_limits=("Release-test fixture only.",),
         ),
     )
 """,
@@ -387,6 +402,7 @@ def replace_with_unscoped_workflow_manifest(repo: Path) -> None:
     manifest = repo / "src" / "govoplan_access" / "backend" / "manifest.py"
     manifest.write_text(
         """from govoplan_core.core.modules import DocumentationTopic, ModuleManifest
+from govoplan_core.core.provider_governance import declared_module_architecture
 
 
 def get_manifest():
@@ -402,6 +418,20 @@ def get_manifest():
                 documentation_types=("user",),
                 metadata={"kind": "workflow"},
             ),
+            DocumentationTopic(
+                id="access.admin.reference",
+                title="Administer access",
+                summary="Static administrator documentation for the release fixture.",
+                documentation_types=("admin",),
+                metadata={"kind": "reference"},
+            ),
+        ),
+        architecture=declared_module_architecture(
+            layer="institutional_foundation",
+            kind="foundation",
+            maturity="scaffold",
+            documentation_ref="pyproject.toml",
+            known_limits=("Release-test fixture only.",),
         ),
     )
 """,

tests/test_runtime_distribution_build.py

@@ -3,10 +3,14 @@ from __future__ import annotations
 import argparse
 import importlib.util
 import json
+import os
 from pathlib import Path
+import shutil
 import sys
 import tempfile
 import unittest
+from unittest.mock import patch
+from urllib.error import HTTPError
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -26,9 +30,211 @@ FINALIZE = _load(
     "finalize_runtime_distribution",
     ROOT / "tools/release/finalize-runtime-distribution.py",
 )
+DEPLOYER_BUILD = _load(
+    "build_deployer_zipapp",
+    ROOT / "tools/deployment/build-deployer-zipapp.py",
+)
+PUBLISH = _load(
+    "publish_runtime_release",
+    ROOT / "tools/release/publish-runtime-release.py",
+)
 
 
 class RuntimeDistributionBuildTests(unittest.TestCase):
+    def test_deployment_zipapp_is_reproducible_across_source_mtimes(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-reproducible-zipapp-") as value:
+            root = Path(value)
+            source = root / "source"
+            shutil.copytree(ROOT / "tools/deployment", source)
+            first = root / "first.pyz"
+            second = root / "second.pyz"
+            original_root = DEPLOYER_BUILD.ROOT
+            try:
+                DEPLOYER_BUILD.ROOT = source
+                self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(first)]))
+                for path in source.rglob("*.py"):
+                    os.utime(path, (2_000_000_000, 2_000_000_000))
+                self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(second)]))
+            finally:
+                DEPLOYER_BUILD.ROOT = original_root
+
+            self.assertEqual(first.read_bytes(), second.read_bytes())
+
+    def test_workflow_signs_with_the_release_environment(self) -> None:
+        workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertIn(
+            ".runtime-build/bin/python tools/release/generate-runtime-distribution.py",
+            workflow,
+        )
+        self.assertNotIn(
+            "\n          python tools/release/generate-runtime-distribution.py",
+            workflow,
+        )
+
+    def test_workflow_rejects_missing_or_mutable_image_inputs_before_build(self) -> None:
+        workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
+            encoding="utf-8"
+        )
+
+        validation = workflow.index("- name: Validate immutable release inputs")
+        bootstrap = workflow.index("- name: Bootstrap release sources")
+        self.assertLess(validation, bootstrap)
+        self.assertIn('image_pattern = re.compile(r"^[^@\\s]+@sha256:', workflow)
+        for input_name in (
+            "python_image",
+            "nginx_image",
+            "postgres_image",
+            "redis_image",
+            "load_balancer_image",
+            "managed_ingress_image",
+            "garage_image",
+            "test_mail_image",
+            "binfmt_image",
+        ):
+            self.assertIn(f"inputs.{input_name}", workflow)
+
+    def test_api_runtime_points_core_at_packaged_migration_scripts(self) -> None:
+        dockerfile = (ROOT / "tools/release/runtime/Dockerfile.api").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertIn(
+            "GOVOPLAN_CORE_SOURCE_ROOT=/opt/govoplan/runtime/govoplan_core_runtime",
+            dockerfile,
+        )
+
+    def test_web_runtime_uses_only_writable_tmpfs_for_nginx_temp_files(self) -> None:
+        nginx = (ROOT / "tools/release/runtime/nginx.conf").read_text(
+            encoding="utf-8"
+        )
+
+        for temporary_path in (
+            "client_body_temp_path /tmp/client_temp;",
+            "fastcgi_temp_path /tmp/fastcgi_temp;",
+            "proxy_temp_path /tmp/proxy_temp;",
+            "scgi_temp_path /tmp/scgi_temp;",
+            "uwsgi_temp_path /tmp/uwsgi_temp;",
+        ):
+            self.assertIn(temporary_path, nginx)
+
+    def test_workflow_verifies_portable_bootstrap_artifacts_before_execution(
+        self,
+    ) -> None:
+        workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertIn(
+            "(cd runtime-output && sha256sum govoplan-deploy.pyz > "
+            "govoplan-deploy.pyz.sha256)",
+            workflow,
+        )
+        self.assertIn("openssl pkeyutl -verify -pubin", workflow)
+        self.assertIn("govoplan-deploy.tampered.pyz", workflow)
+        self.assertLess(
+            workflow.index("openssl pkeyutl -verify -pubin"),
+            workflow.index("python runtime-output/govoplan-deploy.pyz init"),
+        )
+
+    def test_workflow_retains_both_platform_runtime_smoke_receipts(self) -> None:
+        workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertIn('for ARCH in amd64 arm64; do', workflow)
+        self.assertIn("tools/checks/runtime-image-smoke.py", workflow)
+        self.assertIn("Resolve managed dependency platform images", workflow)
+        self.assertIn("--postgres-metadata", workflow)
+        self.assertIn("--redis-metadata", workflow)
+        self.assertIn("Register arm64 execution for runtime smoke", workflow)
+        self.assertIn(
+            'docker run --privileged --rm "$BINFMT_IMAGE" --install arm64',
+            workflow,
+        )
+        self.assertIn("runtime-smoke-amd64.json", workflow)
+        self.assertIn("runtime-smoke-arm64.json", workflow)
+        self.assertIn(
+            "jq -r '.platforms[\"linux/amd64\"]' runtime-output/api-metadata.json",
+            workflow,
+        )
+        self.assertNotIn(".platforms[\\\"linux/amd64\\\"]", workflow)
+
+    def test_workflow_binds_the_release_tag_to_the_workflow_commit(self) -> None:
+        workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
+            encoding="utf-8"
+        )
+        publisher = (ROOT / "tools/release/publish-runtime-release.py").read_text(
+            encoding="utf-8"
+        )
+
+        self.assertIn("SOURCE_COMMIT: ${{ gitea.sha }}", workflow)
+        self.assertIn('--target-commit "$SOURCE_COMMIT"', workflow)
+        self.assertIn('"target_commitish": target_commit', publisher)
+        self.assertIn("self._resolve_commit(tag) != target_commit", publisher)
+
+    def test_runtime_publisher_rejects_a_tag_on_another_commit(self) -> None:
+        publisher = PUBLISH.GiteaReleasePublisher(
+            base_url="https://git.example.test",
+            owner="GovOPlaN",
+            repo="govoplan",
+            token="secret",
+        )
+        target = "1" * 40
+        with (
+            patch.object(
+                publisher,
+                "_resolve_commit",
+                side_effect=(target, "2" * 40),
+            ),
+            self.assertRaisesRegex(PUBLISH.PublishError, "another commit"),
+        ):
+            publisher.release(
+                tag="v1.2.3",
+                target_commit=target,
+                title="Release",
+                body="Body",
+            )
+
+    def test_runtime_publisher_creates_the_tag_at_the_exact_commit(self) -> None:
+        publisher = PUBLISH.GiteaReleasePublisher(
+            base_url="https://git.example.test",
+            owner="GovOPlaN",
+            repo="govoplan",
+            token="secret",
+        )
+        target = "1" * 40
+        requests: list[tuple[str, dict[str, object] | None]] = []
+
+        def request(method: str, _url: str, **kwargs):
+            payload = kwargs.get("payload")
+            requests.append((method, payload))
+            if method == "GET":
+                raise HTTPError(_url, 404, "not found", {}, None)
+            return {"id": 1}
+
+        with (
+            patch.object(
+                publisher,
+                "_resolve_commit",
+                side_effect=(target, None, target),
+            ),
+            patch.object(publisher, "_json", side_effect=request),
+        ):
+            release = publisher.release(
+                tag="v1.2.3",
+                target_commit=target,
+                title="Release",
+                body="Body",
+            )
+
+        self.assertEqual({"id": 1}, release)
+        self.assertEqual("POST", requests[-1][0])
+        assert requests[-1][1] is not None
+        self.assertEqual(target, requests[-1][1]["target_commitish"])
+
     def test_resolves_platforms_and_builds_evidence_descriptor(self) -> None:
         index = {
             "schemaVersion": 2,
@@ -52,6 +258,15 @@ class RuntimeDistributionBuildTests(unittest.TestCase):
             "registry.example/govoplan/api@sha256:" + "1" * 64,
             metadata["platforms"]["linux/amd64"],
         )
+        dependency_metadata = OCI.resolve_platforms(
+            index,
+            repository="registry.example:5000/library/postgres:16-alpine",
+            index_digest="sha256:" + "a" * 64,
+        )
+        self.assertEqual(
+            "registry.example:5000/library/postgres@sha256:" + "2" * 64,
+            dependency_metadata["platforms"]["linux/arm64"],
+        )
 
         with tempfile.TemporaryDirectory(prefix="govoplan-runtime-finalize-") as value:
             root = Path(value)

tests/test_runtime_image_smoke.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "tools/checks/runtime-image-smoke.py"
+SPEC = importlib.util.spec_from_file_location("runtime_image_smoke", SCRIPT)
+assert SPEC is not None and SPEC.loader is not None
+MODULE = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = MODULE
+SPEC.loader.exec_module(MODULE)
+
+
+class RuntimeImageSmokeTests(unittest.TestCase):
+    def test_selects_the_exact_platform_digest(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value:
+            path = Path(value) / "metadata.json"
+            path.write_text(
+                json.dumps(
+                    {
+                        "index": "registry.example/api@sha256:" + "a" * 64,
+                        "platforms": {
+                            "linux/amd64": "registry.example/api@sha256:" + "1" * 64,
+                            "linux/arm64": "registry.example/api@sha256:" + "2" * 64,
+                        },
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            self.assertEqual(
+                "registry.example/api@sha256:" + "2" * 64,
+                MODULE.platform_image(path, "linux/arm64", "API"),
+            )
+
+    def test_rejects_mutable_or_missing_platform_images(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value:
+            path = Path(value) / "metadata.json"
+            path.write_text(
+                json.dumps({"platforms": {"linux/amd64": "registry.example/api:latest"}}),
+                encoding="utf-8",
+            )
+
+            with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"):
+                MODULE.platform_image(path, "linux/amd64", "API")
+            with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"):
+                MODULE.platform_image(path, "linux/arm64", "API")
+
+    def test_readiness_fails_immediately_when_container_exits(self) -> None:
+        exited = subprocess.CompletedProcess([], 0, "false\n", "")
+        logs = subprocess.CompletedProcess(
+            [], 0, "fatal startup error db-secret\n", ""
+        )
+        with patch.object(MODULE, "_run", side_effect=(exited, logs)):
+            with self.assertRaisesRegex(
+                MODULE.SmokeError,
+                r"container exited before readiness: fatal startup error \[redacted\]",
+            ):
+                MODULE._wait_for(
+                    "WebUI",
+                    lambda: self.fail("probe must not run for an exited container"),
+                    timeout=60,
+                    container="web",
+                    redactions=("db-secret",),
+                )
+
+    def test_smoke_supplies_the_packaged_web_upstream_and_schema_contract(self) -> None:
+        source = SCRIPT.read_text(encoding="utf-8")
+
+        self.assertIn('"--network-alias",\n                "load-balancer"', source)
+        self.assertIn("'core_system_settings'", source)
+        self.assertIn("'core_runtime_nodes'", source)
+        self.assertIn('if platform == "linux/arm64"', source)
+        self.assertIn('"ARM64-COW-BUG"', source)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/checks/check-datasource-composition.py

@@ -25,6 +25,12 @@ from govoplan_core.core.datasources import (
     datasource_publication,
 )
 from govoplan_core.core.modules import ModuleContext
+from govoplan_core.core.recovery import RecoveryCheckpoint, RecoveryOperation
+from govoplan_core.core.runtime_coordination import (
+    DistributedLease,
+    RuntimeIdentity,
+    bind_process_runtime_identity,
+)
 from govoplan_core.core.tabular_sources import (
     TabularSnapshotInput,
     tabular_snapshot_writer,
@@ -73,6 +79,9 @@ def main() -> int:
     Base.metadata.create_all(
         engine,
         tables=[
+            DistributedLease.__table__,
+            RecoveryOperation.__table__,
+            RecoveryCheckpoint.__table__,
             ConnectorTabularSource.__table__,
             DatasourceRecord.__table__,
             DatasourcePayloadRecord.__table__,
@@ -86,153 +95,168 @@ def main() -> int:
         ],
     )
     session_factory = sessionmaker(bind=engine)
-    with session_factory() as session:
-        principal = _principal()
-        writer = tabular_snapshot_writer(registry)
-        lifecycle = datasource_lifecycle(registry)
-        catalogue = datasource_catalogue(registry)
-        publisher = datasource_publication(registry)
-        runner = dataflow_run_lifecycle(registry)
-        if (
-            writer is None
-            or lifecycle is None
-            or catalogue is None
-            or publisher is None
-            or runner is None
-        ):
-            raise RuntimeError("Datasource composition capabilities are incomplete.")
+    bind_process_runtime_identity(_runtime_identity())
+    try:
+        with session_factory() as session:
+            principal = _principal()
+            writer = tabular_snapshot_writer(registry)
+            lifecycle = datasource_lifecycle(registry)
+            catalogue = datasource_catalogue(registry)
+            publisher = datasource_publication(registry)
+            runner = dataflow_run_lifecycle(registry)
+            if (
+                writer is None
+                or lifecycle is None
+                or catalogue is None
+                or publisher is None
+                or runner is None
+            ):
+                raise RuntimeError(
+                    "Datasource composition capabilities are incomplete."
+                )
 
-        origin = writer.create_snapshot(
-            session,
-            principal,
-            snapshot=TabularSnapshotInput(
-                name="Monthly cases",
-                source_name="connector_monthly_cases",
-                rows=(
-                    {"id": 1, "amount": 5},
-                    {"id": 2, "amount": 15},
+            origin = writer.create_snapshot(
+                session,
+                principal,
+                snapshot=TabularSnapshotInput(
+                    name="Monthly cases",
+                    source_name="connector_monthly_cases",
+                    rows=(
+                        {"id": 1, "amount": 5},
+                        {"id": 2, "amount": 15},
+                    ),
                 ),
-            ),
-        )
-        datasource = lifecycle.register_origin(
-            session,
-            principal,
-            origin_ref=origin.ref,
-            name="Monthly cases cache",
-            source_name="monthly_cases",
-            mode="cached",
-        )
-        result = preview_pipeline(
-            session,
-            tenant_id="tenant-1",
-            actor_id="account-1",
-            payload=PipelinePreviewRequest(
-                graph=_graph(
-                    datasource_ref=datasource.ref,
-                    fingerprint=datasource.fingerprint,
+            )
+            datasource = lifecycle.register_origin(
+                session,
+                principal,
+                origin_ref=origin.ref,
+                name="Monthly cases cache",
+                source_name="monthly_cases",
+                mode="cached",
+            )
+            result = preview_pipeline(
+                session,
+                tenant_id="tenant-1",
+                actor_id="account-1",
+                payload=PipelinePreviewRequest(
+                    graph=_graph(
+                        datasource_ref=datasource.ref,
+                        fingerprint=datasource.fingerprint,
+                    ),
+                    row_limit=100,
                 ),
-                row_limit=100,
-            ),
-            principal=principal,
-            registry=registry,
-        )
-        expected_rows = [
-            {"id": 1, "amount": 5},
-            {"id": 2, "amount": 15},
-        ]
-        if result.status != "succeeded":
-            raise RuntimeError(f"Dataflow preview failed: {result.diagnostics}")
-        if result.rows != expected_rows:
-            raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
-        if result.source_fingerprints[0]["source_ref"] != datasource.ref:
-            raise RuntimeError("Dataflow lineage did not retain the datasource reference.")
-        pipeline = create_pipeline(
-            session,
-            tenant_id="tenant-1",
-            actor_id="account-1",
-            payload=PipelineCreateRequest(
-                name="Monthly case output",
-                status="active",
-                graph=_graph(
-                    datasource_ref=datasource.ref,
-                    fingerprint=datasource.fingerprint,
+                principal=principal,
+                registry=registry,
+            )
+            expected_rows = [
+                {"id": 1, "amount": 5},
+                {"id": 2, "amount": 15},
+            ]
+            if result.status != "succeeded":
+                raise RuntimeError(
+                    f"Dataflow preview failed: {result.diagnostics}"
+                )
+            if result.rows != expected_rows:
+                raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
+            if result.source_fingerprints[0]["source_ref"] != datasource.ref:
+                raise RuntimeError(
+                    "Dataflow lineage did not retain the datasource reference."
+                )
+            pipeline = create_pipeline(
+                session,
+                tenant_id="tenant-1",
+                actor_id="account-1",
+                payload=PipelineCreateRequest(
+                    name="Monthly case output",
+                    status="active",
+                    graph=_graph(
+                        datasource_ref=datasource.ref,
+                        fingerprint=datasource.fingerprint,
+                    ),
+                    editor_mode="graph",
                 ),
-                editor_mode="graph",
-            ),
-        )
-        run_request = DataflowRunRequest(
-            pipeline_ref=f"pipeline:{pipeline.id}",
-            revision=1,
-            idempotency_key="composition-run-1",
-            publication=DataflowPublicationTarget(
-                name="Monthly case result",
-                source_name="monthly_case_result",
-                freeze=True,
-                frozen_label="Composition evidence",
-            ),
-        )
-        published = runner.start_run(
-            session,
-            principal,
-            request=run_request,
-        )
-        replayed = runner.start_run(
-            session,
-            principal,
-            request=run_request,
-        )
-        if published.status != "queued":
-            raise RuntimeError(
-                f"Dataflow run was not queued: {published.status}"
             )
-        worker = SqlDataflowRunWorker(
-            registry=_AutomationRegistry(registry, principal)
-        )
-        worker_result = worker.dispatch_pending(
-            session,
-            worker_id="composition-worker",
-        )
-        if worker_result["succeeded"] != 1:
-            raise RuntimeError(
-                f"Dataflow worker failed: {worker_result!r}"
+            run_request = DataflowRunRequest(
+                pipeline_ref=f"pipeline:{pipeline.id}",
+                revision=1,
+                idempotency_key="composition-run-1",
+                publication=DataflowPublicationTarget(
+                    name="Monthly case result",
+                    source_name="monthly_case_result",
+                    freeze=True,
+                    frozen_label="Composition evidence",
+                ),
             )
-        completed = runner.get_run(
-            session,
-            principal,
-            run_ref=published.ref,
-        )
-        if completed is None:
-            raise RuntimeError("Dataflow run evidence disappeared.")
-        published = completed
-        if published.status != "succeeded":
-            raise RuntimeError(f"Dataflow publication failed: {published.error}")
-        if replayed.ref != published.ref or not replayed.replayed:
-            raise RuntimeError("Dataflow run idempotency did not replay the prior run.")
-        if (
-            not published.output_datasource_ref
-            or not published.output_materialization_ref
-        ):
-            raise RuntimeError("Dataflow publication did not retain output references.")
-        output = catalogue.read_datasource(
-            session,
-            principal,
-            request=DatasourceReadRequest(
-                datasource_ref=published.output_datasource_ref,
-            ),
-        )
-        if list(output.rows) != expected_rows:
-            raise RuntimeError(
-                f"Unexpected published Dataflow rows: {list(output.rows)!r}"
+            published = runner.start_run(
+                session,
+                principal,
+                request=run_request,
             )
-        if (
-            output.materialization is None
-            or output.materialization.ref != published.output_materialization_ref
-            or output.materialization.frozen_at is None
-        ):
-            raise RuntimeError(
-                "Published Datasource materialization is not pinned and frozen."
+            replayed = runner.start_run(
+                session,
+                principal,
+                request=run_request,
             )
-    engine.dispose()
+            if published.status != "queued":
+                raise RuntimeError(
+                    f"Dataflow run was not queued: {published.status}"
+                )
+            worker = SqlDataflowRunWorker(
+                registry=_AutomationRegistry(registry, principal)
+            )
+            worker_result = worker.dispatch_pending(
+                session,
+                worker_id="composition-worker",
+            )
+            if worker_result["succeeded"] != 1:
+                raise RuntimeError(f"Dataflow worker failed: {worker_result!r}")
+            completed = runner.get_run(
+                session,
+                principal,
+                run_ref=published.ref,
+            )
+            if completed is None:
+                raise RuntimeError("Dataflow run evidence disappeared.")
+            published = completed
+            if published.status != "succeeded":
+                raise RuntimeError(
+                    f"Dataflow publication failed: {published.error}"
+                )
+            if replayed.ref != published.ref or not replayed.replayed:
+                raise RuntimeError(
+                    "Dataflow run idempotency did not replay the prior run."
+                )
+            if (
+                not published.output_datasource_ref
+                or not published.output_materialization_ref
+            ):
+                raise RuntimeError(
+                    "Dataflow publication did not retain output references."
+                )
+            output = catalogue.read_datasource(
+                session,
+                principal,
+                request=DatasourceReadRequest(
+                    datasource_ref=published.output_datasource_ref,
+                ),
+            )
+            if list(output.rows) != expected_rows:
+                raise RuntimeError(
+                    f"Unexpected published Dataflow rows: {list(output.rows)!r}"
+                )
+            if (
+                output.materialization is None
+                or output.materialization.ref
+                != published.output_materialization_ref
+                or output.materialization.frozen_at is None
+            ):
+                raise RuntimeError(
+                    "Published Datasource materialization is not pinned and frozen."
+                )
+    finally:
+        bind_process_runtime_identity(None)
+        engine.dispose()
     print(
         "Connector -> Datasources -> pinned Dataflow publication composition passed."
     )
@@ -252,6 +276,17 @@ class _AutomationProvider:
         )
 
 
+def _runtime_identity() -> RuntimeIdentity:
+    return RuntimeIdentity(
+        installation_id="datasource-composition-check",
+        node_id="composition-worker",
+        incarnation="composition-worker-incarnation",
+        role="worker",
+        software_version="test",
+        composition_hash="c" * 64,
+    )
+
+
 class _AutomationRegistry:
     def __init__(self, registry, principal: ApiPrincipal) -> None:
         self.registry = registry

tools/checks/managed-ingress-drill.py

@@ -4,13 +4,14 @@
 from __future__ import annotations
 
 import argparse
+import json
 from pathlib import Path
 import re
 import shutil
+import socket
 import subprocess
 import sys
 import tempfile
-import time
 from uuid import uuid4
 
 
@@ -27,57 +28,211 @@ from govoplan_deploy.model import default_spec  # noqa: E402
 DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
 
 
-def _run(argv: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]:
-    return subprocess.run(
-        argv,
-        check=check,
-        capture_output=True,
-        text=True,
-        timeout=60,
+def _run(
+    argv: list[str],
+    *,
+    check: bool = True,
+    input_text: str | None = None,
+) -> subprocess.CompletedProcess[str]:
+    try:
+        return subprocess.run(
+            argv,
+            check=check,
+            capture_output=True,
+            input=input_text,
+            text=True,
+            timeout=60,
+        )
+    except subprocess.CalledProcessError as exc:
+        stderr = exc.stderr.strip()
+        if stderr:
+            print(stderr, file=sys.stderr)
+        raise
+
+
+def _write_volume_file(
+    *,
+    image: str,
+    volume: str,
+    filename: str,
+    content: str,
+) -> None:
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+", filename):
+        raise ValueError(f"invalid config filename: {filename!r}")
+    _run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "--interactive",
+            "--user",
+            "0:0",
+            "--mount",
+            f"type=volume,src={volume},dst=/govoplan-config",
+            "--entrypoint",
+            "sh",
+            image,
+            "-c",
+            f"umask 022; cat > /govoplan-config/{filename}",
+        ],
+        input_text=content,
     )
 
 
 def _published_port(container: str, target: int) -> int:
-    output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip()
+    output = _run(
+        [
+            "docker",
+            "inspect",
+            "--format",
+            "{{json .HostConfig.PortBindings}}",
+            container,
+        ]
+    ).stdout.strip()
     try:
-        return int(output.rsplit(":", 1)[1])
-    except (IndexError, ValueError) as exc:
-        raise RuntimeError(f"cannot determine published port from {output!r}") from exc
+        bindings = json.loads(output)[f"{target}/tcp"]
+        if not isinstance(bindings, list) or len(bindings) != 1:
+            raise ValueError("expected exactly one published binding")
+        binding = bindings[0]
+        if binding.get("HostIp") != "127.0.0.1":
+            raise ValueError("published binding is not loopback-only")
+        return int(binding["HostPort"])
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+        raise RuntimeError(
+            f"cannot determine loopback binding for {target}/tcp from {output!r}"
+        ) from exc
 
 
-def _curl(url: str, *, headers: bool = False) -> str:
-    argv = ["curl", "--silent", "--show-error", "--insecure"]
-    if headers:
-        argv.extend(["--head"])
-    argv.append(url)
-    return _run(argv).stdout
+def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
+    for _attempt in range(10):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
+            listener.bind(("127.0.0.1", 0))
+            port = int(listener.getsockname()[1])
+        if port not in exclude:
+            return port
+    raise RuntimeError("cannot allocate distinct loopback ports for ingress drill")
 
 
-def _wait_for_https(port: int) -> str:
-    deadline = time.monotonic() + 30
-    last_error = ""
-    while time.monotonic() < deadline:
-        try:
-            return _curl(f"https://localhost:{port}/health")
-        except subprocess.CalledProcessError as exc:
-            last_error = exc.stderr.strip()
-            time.sleep(0.5)
-    raise RuntimeError(f"managed ingress did not become ready: {last_error}")
+def _probe_ingress(*, image: str, network: str, container: str) -> None:
+    probe = r'''
+import socket
+import ssl
+import time
+
+
+def request(port, payload, *, tls):
+    connection = socket.create_connection(("ingress", port), timeout=3)
+    if tls:
+        connection = ssl._create_unverified_context().wrap_socket(
+            connection, server_hostname="localhost"
+        )
+    with connection:
+        connection.sendall(payload)
+        chunks = []
+        while True:
+            chunk = connection.recv(65536)
+            if not chunk:
+                break
+            chunks.append(chunk)
+    return b"".join(chunks)
+
+
+deadline = time.monotonic() + 30
+last_error = ""
+while time.monotonic() < deadline:
+    try:
+        response = request(
+            8443,
+            b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+            tls=True,
+        )
+        head, body_bytes = response.split(b"\r\n\r\n", 1)
+        status = int(head.split(b" ", 2)[1])
+        if status != 200:
+            raise RuntimeError(f"HTTPS returned {status}, expected 200")
+        body = body_bytes.decode("utf-8").strip()
+        if body != "proto=https":
+            raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
+        break
+    except Exception as exc:
+        last_error = f"{type(exc).__name__}: {exc}"
+        time.sleep(0.5)
+else:
+    raise SystemExit(f"managed ingress did not become ready: {last_error}")
+
+response = request(
+    8080,
+    b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
+    tls=False,
+)
+head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1")
+lines = head.split("\r\n")
+status = int(lines[0].split(" ", 2)[1])
+if status != 308:
+    raise SystemExit(f"HTTP returned {status}, expected redirect 308")
+headers = {
+    key.lower(): value.strip()
+    for key, separator, value in (line.partition(":") for line in lines[1:])
+    if separator
+}
+location = headers.get("location", "")
+if not location.startswith("https://localhost"):
+    raise SystemExit(f"HTTP redirect had unexpected location: {location!r}")
+'''
+    try:
+        _run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--network",
+                network,
+                "--read-only",
+                "--security-opt",
+                "no-new-privileges",
+                "--cap-drop",
+                "ALL",
+                "--entrypoint",
+                "python",
+                image,
+                "-c",
+                probe,
+            ]
+        )
+    except subprocess.CalledProcessError:
+        _print_container_diagnostics(container)
+        raise
+
+
+def _print_container_diagnostics(container: str) -> None:
+    state = _run(
+        ["docker", "inspect", "--format", "{{json .State}}", container],
+        check=False,
+    )
+    state_detail = (state.stdout + state.stderr).strip()
+    if state_detail:
+        print(f"managed ingress state:\n{state_detail}", file=sys.stderr)
+    logs = _run(["docker", "logs", container], check=False)
+    log_detail = (logs.stdout + logs.stderr).strip()
+    if log_detail:
+        print(f"managed ingress logs:\n{log_detail}", file=sys.stderr)
 
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--caddy-image", required=True)
     parser.add_argument("--load-balancer-image", required=True)
+    parser.add_argument("--probe-image", required=True)
     args = parser.parse_args()
     for label, image in (
         ("--caddy-image", args.caddy_image),
         ("--load-balancer-image", args.load_balancer_image),
+        ("--probe-image", args.probe_image),
     ):
         if DIGEST_IMAGE.fullmatch(image) is None:
             parser.error(f"{label} must be pinned by sha256 digest")
-    if shutil.which("docker") is None or shutil.which("curl") is None:
-        parser.error("docker and curl are required")
+    if shutil.which("docker") is None:
+        parser.error("docker is required")
 
     suffix = uuid4().hex[:10]
     network = f"govoplan-ingress-drill-{suffix}"
@@ -85,15 +240,33 @@ def main() -> int:
     backend = f"govoplan-ingress-backend-{suffix}"
     data_volume = f"govoplan-ingress-data-{suffix}"
     config_volume = f"govoplan-ingress-config-{suffix}"
+    backend_config_volume = f"govoplan-ingress-backend-config-{suffix}"
+    ingress_config_volume = f"govoplan-ingress-caddy-config-{suffix}"
+    load_balancer_config_volume = f"govoplan-ingress-haproxy-config-{suffix}"
     cleanup = [
         ["docker", "rm", "--force", ingress, backend],
         ["docker", "network", "rm", network],
-        ["docker", "volume", "rm", data_volume, config_volume],
+        [
+            "docker",
+            "volume",
+            "rm",
+            data_volume,
+            config_volume,
+            backend_config_volume,
+            ingress_config_volume,
+            load_balancer_config_volume,
+        ],
     ]
     try:
         _run(["docker", "network", "create", network])
-        _run(["docker", "volume", "create", data_volume])
-        _run(["docker", "volume", "create", config_volume])
+        for volume in (
+            data_volume,
+            config_volume,
+            backend_config_volume,
+            ingress_config_volume,
+            load_balancer_config_volume,
+        ):
+            _run(["docker", "volume", "create", volume])
         with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory:
             root = Path(directory)
             backend_config = root / "backend.Caddyfile"
@@ -127,6 +300,24 @@ def main() -> int:
                 encoding="utf-8",
             )
             load_balancer_config.chmod(0o644)
+            _write_volume_file(
+                image=args.load_balancer_image,
+                volume=load_balancer_config_volume,
+                filename="haproxy.cfg",
+                content=load_balancer_config.read_text(encoding="utf-8"),
+            )
+            _write_volume_file(
+                image=args.caddy_image,
+                volume=backend_config_volume,
+                filename="Caddyfile",
+                content=backend_config.read_text(encoding="utf-8"),
+            )
+            _write_volume_file(
+                image=args.caddy_image,
+                volume=ingress_config_volume,
+                filename="Caddyfile",
+                content=ingress_config.read_text(encoding="utf-8"),
+            )
             _run(
                 [
                     "docker",
@@ -136,7 +327,11 @@ def main() -> int:
                     "--cap-drop",
                     "ALL",
                     "--mount",
-                    f"type=bind,src={load_balancer_config},dst=/usr/local/etc/haproxy/haproxy.cfg,readonly",
+                    (
+                        "type=volume,"
+                        f"src={load_balancer_config_volume},"
+                        "dst=/usr/local/etc/haproxy,readonly"
+                    ),
                     args.load_balancer_image,
                     "haproxy",
                     "-c",
@@ -154,18 +349,28 @@ def main() -> int:
                     backend,
                     "--network",
                     network,
+                    "--network-alias",
+                    "load-balancer",
                     "--read-only",
                     "--tmpfs",
                     "/tmp:rw,noexec,nosuid,size=16m",
                     "--mount",
-                    f"type=bind,src={backend_config},dst=/etc/caddy/Caddyfile,readonly",
+                    (
+                        "type=volume,"
+                        f"src={backend_config_volume},"
+                        "dst=/govoplan-config,readonly"
+                    ),
                     args.caddy_image,
                     "caddy",
                     "run",
                     "--config",
-                    "/etc/caddy/Caddyfile",
+                    "/govoplan-config/Caddyfile",
                 ]
             )
+            requested_http_port = _available_loopback_port()
+            requested_https_port = _available_loopback_port(
+                exclude=frozenset({requested_http_port})
+            )
             ingress_command = [
                 "docker",
                 "run",
@@ -174,6 +379,8 @@ def main() -> int:
                 ingress,
                 "--network",
                 network,
+                "--network-alias",
+                "ingress",
                 "--read-only",
                 "--tmpfs",
                 "/tmp:rw,noexec,nosuid,size=16m",
@@ -181,12 +388,18 @@ def main() -> int:
                 "no-new-privileges",
                 "--cap-drop",
                 "ALL",
+                "--cap-add",
+                "NET_BIND_SERVICE",
                 "--publish",
-                "127.0.0.1::8080",
+                f"127.0.0.1:{requested_http_port}:8080/tcp",
                 "--publish",
-                "127.0.0.1::8443",
+                f"127.0.0.1:{requested_https_port}:8443/tcp",
                 "--mount",
-                f"type=bind,src={ingress_config},dst=/etc/caddy/Caddyfile,readonly",
+                (
+                    "type=volume,"
+                    f"src={ingress_config_volume},"
+                    "dst=/govoplan-config,readonly"
+                ),
                 "--mount",
                 f"type=volume,src={data_volume},dst=/data",
                 "--mount",
@@ -195,25 +408,32 @@ def main() -> int:
                 "caddy",
                 "run",
                 "--config",
-                "/etc/caddy/Caddyfile",
+                "/govoplan-config/Caddyfile",
             ]
             _run(ingress_command)
             http_port = _published_port(ingress, 8080)
             https_port = _published_port(ingress, 8443)
-            body = _wait_for_https(https_port)
-            if body.strip() != "proto=https":
-                raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
-            redirect = _curl(f"http://localhost:{http_port}/health", headers=True)
-            if not redirect.startswith("HTTP/1.1 308"):
-                raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}")
+            if (http_port, https_port) != (
+                requested_http_port,
+                requested_https_port,
+            ):
+                raise RuntimeError("Docker published unexpected ingress ports")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
 
             _run(["docker", "rm", "--force", ingress])
             _run(ingress_command)
             https_port = _published_port(ingress, 8443)
-            if _wait_for_https(https_port).strip() != "proto=https":
-                raise RuntimeError(
-                    "managed ingress did not recover with persistent state"
-                )
+            if https_port != requested_https_port:
+                raise RuntimeError("Docker changed the ingress TLS binding on restart")
+            _probe_ingress(
+                image=args.probe_image,
+                network=network,
+                container=ingress,
+            )
             _run(
                 [
                     "docker",

tools/checks/runtime-image-smoke.py

@@ -0,0 +1,656 @@
+#!/usr/bin/env python3
+"""Exercise a pinned GovOPlaN runtime image pair on one OCI platform."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+from datetime import UTC, datetime
+import json
+import os
+from pathlib import Path
+import re
+import secrets
+import subprocess
+import time
+from typing import Callable, Sequence
+
+
+PLATFORMS = frozenset({"linux/amd64", "linux/arm64"})
+DIGEST_IMAGE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$")
+BASE_MODULES = (
+    "tenancy",
+    "organizations",
+    "identity",
+    "idm",
+    "access",
+    "admin",
+    "dashboard",
+    "policy",
+    "audit",
+    "docs",
+    "ops",
+)
+
+
+class SmokeError(RuntimeError):
+    """A runtime image failed its bounded acceptance drill."""
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--api-metadata", type=Path, required=True)
+    parser.add_argument("--web-metadata", type=Path, required=True)
+    parser.add_argument("--postgres-metadata", type=Path, required=True)
+    parser.add_argument("--redis-metadata", type=Path, required=True)
+    parser.add_argument("--platform", choices=sorted(PLATFORMS), required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--timeout-seconds", type=float, default=600.0)
+    return parser
+
+
+def _utc_now() -> str:
+    return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _digest_image(value: object, label: str) -> str:
+    if not isinstance(value, str) or DIGEST_IMAGE.fullmatch(value) is None:
+        raise SmokeError(f"{label} must be an exact sha256 image reference")
+    return value
+
+
+def platform_image(path: Path, platform: str, label: str) -> str:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise SmokeError(f"cannot read {label} OCI metadata") from exc
+    if not isinstance(payload, dict) or not isinstance(payload.get("platforms"), dict):
+        raise SmokeError(f"{label} OCI metadata has no platform map")
+    return _digest_image(payload["platforms"].get(platform), f"{label} {platform}")
+
+
+def _tail(value: str, *, limit: int = 4000) -> str:
+    return value[-limit:].strip()
+
+
+def _run(
+    arguments: Sequence[str],
+    *,
+    check: bool = True,
+    timeout: float = 600.0,
+    redactions: Sequence[str] = (),
+) -> subprocess.CompletedProcess[str]:
+    try:
+        result = subprocess.run(
+            list(arguments),
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise SmokeError(f"container command could not complete: {type(exc).__name__}") from exc
+    if check and result.returncode != 0:
+        detail = _tail(result.stderr or result.stdout or "no diagnostic output")
+        for secret in redactions:
+            if secret:
+                detail = detail.replace(secret, "[redacted]")
+        raise SmokeError(f"container command failed: {detail}")
+    return result
+
+
+def _wait_for(
+    label: str,
+    probe: Callable[[], subprocess.CompletedProcess[str]],
+    *,
+    timeout: float,
+    container: str | None = None,
+    redactions: Sequence[str] = (),
+) -> None:
+    deadline = time.monotonic() + timeout
+    last = ""
+    while time.monotonic() < deadline:
+        if container is not None:
+            state = _run(
+                ("docker", "inspect", "--format", "{{.State.Running}}", container),
+                check=False,
+                timeout=30,
+            )
+            if state.returncode != 0 or state.stdout.strip() != "true":
+                logs = _run(
+                    ("docker", "logs", "--tail", "100", container),
+                    check=False,
+                    timeout=30,
+                )
+                detail = _tail(logs.stdout + logs.stderr, limit=8000)
+                for secret in redactions:
+                    if secret:
+                        detail = detail.replace(secret, "[redacted]")
+                raise SmokeError(
+                    f"{label} container exited before readiness: "
+                    f"{detail or 'no diagnostic output'}"
+                )
+        result = probe()
+        if result.returncode == 0:
+            return
+        last = _tail(result.stderr or result.stdout)
+        time.sleep(2.0)
+    raise SmokeError(f"{label} did not become ready: {last or 'probe failed'}")
+
+
+def _environment(
+    *,
+    database_password: str,
+    master_key: str,
+    platform_slug: str,
+) -> dict[str, str]:
+    database = (
+        f"postgresql+psycopg://govoplan:{database_password}@postgres:5432/govoplan"
+    )
+    return {
+        "APP_ENV": "dev",
+        "GOVOPLAN_INSTALL_PROFILE": "evaluation",
+        "GOVOPLAN_INSTALLATION_ID": f"runtime-smoke-{platform_slug}",
+        "GOVOPLAN_STATE_PROFILE": "host-shared",
+        "GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "5",
+        "GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "30",
+        "GOVOPLAN_EXPECTED_API_REPLICAS": "1",
+        "GOVOPLAN_EXPECTED_WORKER_REPLICAS": "1",
+        "DATABASE_URL": database,
+        "GOVOPLAN_DATABASE_URL_PGTOOLS": (
+            f"postgresql://govoplan:{database_password}@postgres:5432/govoplan"
+        ),
+        "GOVOPLAN_DB_CONNECTION_LIMIT": "100",
+        "GOVOPLAN_DB_CONNECTION_RESERVE": "10",
+        "REDIS_URL": "redis://redis:6379/0",
+        "CELERY_ENABLED": "true",
+        "CELERY_QUEUES": "default",
+        "CELERY_WORKER_CONCURRENCY": "1",
+        "ENABLED_MODULES": ",".join(BASE_MODULES),
+        "GOVOPLAN_MIGRATION_TRACK": "release",
+        "DEV_AUTO_MIGRATE_ENABLED": "false",
+        "DEV_BOOTSTRAP_ENABLED": "false",
+        "AUTH_LOGIN_THROTTLE_ENABLED": "true",
+        "AUTH_COOKIE_SECURE": "false",
+        "CORS_ORIGINS": "http://localhost",
+        "GOVOPLAN_TRUSTED_HOSTS": "127.0.0.1,localhost,api",
+        "FORWARDED_ALLOW_IPS": "127.0.0.1",
+        "MASTER_KEY_B64": master_key,
+        "FILE_STORAGE_BACKEND": "local",
+        "FILE_STORAGE_LOCAL_ROOT": "/var/lib/govoplan/files",
+        "GOVOPLAN_MODULE_LIVE_APPLY_ENABLED": "false",
+    }
+
+
+def _env_arguments(values: dict[str, str], *, role: str, node_id: str) -> list[str]:
+    arguments: list[str] = []
+    for key, value in sorted(
+        {**values, "GOVOPLAN_RUNTIME_ROLE": role, "GOVOPLAN_NODE_ID": node_id}.items()
+    ):
+        arguments.extend(("--env", f"{key}={value}"))
+    return arguments
+
+
+def run_smoke(
+    *,
+    api_image: str,
+    web_image: str,
+    postgres_image: str,
+    redis_image: str,
+    platform: str,
+    timeout: float,
+) -> dict[str, object]:
+    for label, value in (
+        ("API image", api_image),
+        ("Web image", web_image),
+        ("PostgreSQL image", postgres_image),
+        ("Redis image", redis_image),
+    ):
+        _digest_image(value, label)
+    if platform not in PLATFORMS:
+        raise SmokeError(f"unsupported runtime smoke platform: {platform}")
+
+    slug = platform.replace("linux/", "").replace("/", "-")
+    suffix = secrets.token_hex(4)
+    prefix = f"govoplan-runtime-{slug}-{suffix}"
+    names = {
+        "network": f"{prefix}-network",
+        "volume": f"{prefix}-data",
+        "postgres": f"{prefix}-postgres",
+        "redis": f"{prefix}-redis",
+        "api": f"{prefix}-api",
+        "web": f"{prefix}-web",
+        "worker": f"{prefix}-worker",
+    }
+    database_password = secrets.token_hex(20)
+    master_key = base64.urlsafe_b64encode(os.urandom(32)).decode("ascii")
+    redactions = (database_password, master_key)
+    environment = _environment(
+        database_password=database_password,
+        master_key=master_key,
+        platform_slug=slug,
+    )
+    checks: list[dict[str, object]] = []
+    started = time.monotonic()
+
+    def record(check_id: str, began: float) -> None:
+        duration = round(time.monotonic() - began, 3)
+        checks.append(
+            {
+                "id": check_id,
+                "state": "passed",
+                "duration_seconds": duration,
+            }
+        )
+        print(f"PASS {platform} {check_id} ({duration}s)", flush=True)
+
+    common_runtime = [
+        "--platform",
+        platform,
+        "--network",
+        names["network"],
+        "--read-only",
+        "--tmpfs",
+        "/tmp:rw,noexec,nosuid,size=64m",
+        "--security-opt",
+        "no-new-privileges:true",
+        "--cap-drop",
+        "ALL",
+        "--mount",
+        f"type=volume,source={names['volume']},target=/var/lib/govoplan",
+    ]
+    redis_command = ["redis-server", "--save", "", "--appendonly", "no"]
+    if platform == "linux/arm64":
+        # QEMU user-mode execution triggers Redis's host-kernel COW guard even
+        # though this isolated smoke disables every persistence mechanism.
+        redis_command.extend(("--ignore-warnings", "ARM64-COW-BUG"))
+
+    try:
+        _run(("docker", "network", "create", names["network"]), timeout=timeout)
+        _run(("docker", "volume", "create", names["volume"]), timeout=timeout)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--detach",
+                "--platform",
+                platform,
+                "--name",
+                names["postgres"],
+                "--network",
+                names["network"],
+                "--network-alias",
+                "postgres",
+                "--env",
+                "POSTGRES_DB=govoplan",
+                "--env",
+                "POSTGRES_USER=govoplan",
+                "--env",
+                f"POSTGRES_PASSWORD={database_password}",
+                "--tmpfs",
+                "/var/lib/postgresql/data:rw,noexec,nosuid,size=384m",
+                postgres_image,
+            ),
+            timeout=timeout,
+            redactions=redactions,
+        )
+        _run(
+            (
+                "docker",
+                "run",
+                "--detach",
+                "--platform",
+                platform,
+                "--name",
+                names["redis"],
+                "--network",
+                names["network"],
+                "--network-alias",
+                "redis",
+                "--read-only",
+                "--tmpfs",
+                "/data:rw,noexec,nosuid,size=64m",
+                redis_image,
+                *redis_command,
+            ),
+            timeout=timeout,
+        )
+        _wait_for(
+            "PostgreSQL",
+            lambda: _run(
+                (
+                    "docker",
+                    "exec",
+                    names["postgres"],
+                    "pg_isready",
+                    "--username",
+                    "govoplan",
+                    "--dbname",
+                    "govoplan",
+                ),
+                check=False,
+                timeout=30,
+            ),
+            timeout=timeout,
+            container=names["postgres"],
+            redactions=redactions,
+        )
+        _wait_for(
+            "Redis",
+            lambda: _run(
+                ("docker", "exec", names["redis"], "redis-cli", "ping"),
+                check=False,
+                timeout=30,
+            ),
+            timeout=timeout,
+            container=names["redis"],
+            redactions=redactions,
+        )
+        record("managed_dependencies_ready", began)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--rm",
+                "--name",
+                f"{prefix}-migrate",
+                *common_runtime,
+                *_env_arguments(
+                    environment,
+                    role="migration",
+                    node_id=f"runtime-smoke-{slug}-migration",
+                ),
+                api_image,
+                "python",
+                "-m",
+                "govoplan_core.commands.init_db",
+                "--migration-track",
+                "release",
+            ),
+            timeout=timeout,
+            redactions=redactions,
+        )
+        record("release_migrations", began)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--rm",
+                "--name",
+                f"{prefix}-schema",
+                *common_runtime,
+                *_env_arguments(
+                    environment,
+                    role="migration",
+                    node_id=f"runtime-smoke-{slug}-schema",
+                ),
+                api_image,
+                "python",
+                "-c",
+                (
+                    "import os;"
+                    "from sqlalchemy import create_engine,inspect;"
+                    "engine=create_engine(os.environ['DATABASE_URL']);"
+                    "tables=set(inspect(engine).get_table_names());"
+                    "required={'alembic_version','core_scopes','core_system_settings',"
+                    "'core_runtime_nodes'};"
+                    "missing=required-tables;"
+                    "assert not missing, f'missing release tables: {sorted(missing)}';"
+                    "engine.dispose()"
+                ),
+            ),
+            timeout=timeout,
+            redactions=redactions,
+        )
+        record("release_schema_contract", began)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--detach",
+                "--name",
+                names["api"],
+                "--network-alias",
+                "api",
+                "--network-alias",
+                "load-balancer",
+                *common_runtime,
+                *_env_arguments(
+                    environment,
+                    role="api",
+                    node_id=f"runtime-smoke-{slug}-api",
+                ),
+                api_image,
+            ),
+            timeout=timeout,
+            redactions=redactions,
+        )
+        _wait_for(
+            "GovOPlaN API",
+            lambda: _run(
+                (
+                    "docker",
+                    "exec",
+                    names["api"],
+                    "python",
+                    "-c",
+                    (
+                        "import urllib.request;"
+                        "r=urllib.request.Request('http://127.0.0.1:8000/health/ready',"
+                        "headers={'Host':'127.0.0.1'});"
+                        "assert urllib.request.urlopen(r,timeout=3).status==200"
+                    ),
+                ),
+                check=False,
+                timeout=30,
+            ),
+            timeout=timeout,
+            container=names["api"],
+            redactions=redactions,
+        )
+        _run(
+            (
+                "docker",
+                "exec",
+                names["api"],
+                "python",
+                "-c",
+                "import os; assert os.getuid() == 10001",
+            ),
+            timeout=30,
+        )
+        record("api_non_root_readiness", began)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--detach",
+                "--platform",
+                platform,
+                "--name",
+                names["web"],
+                "--network",
+                names["network"],
+                "--network-alias",
+                "web",
+                "--read-only",
+                "--tmpfs",
+                "/tmp:rw,noexec,nosuid,size=64m",
+                "--security-opt",
+                "no-new-privileges:true",
+                "--cap-drop",
+                "ALL",
+                web_image,
+            ),
+            timeout=timeout,
+        )
+        _wait_for(
+            "GovOPlaN WebUI",
+            lambda: _run(
+                (
+                    "docker",
+                    "exec",
+                    names["api"],
+                    "python",
+                    "-c",
+                    (
+                        "import urllib.request;"
+                        "assert urllib.request.urlopen('http://web:8080/health',timeout=3).status==200;"
+                        "assert urllib.request.urlopen('http://web:8080/',timeout=3).status==200"
+                    ),
+                ),
+                check=False,
+                timeout=30,
+            ),
+            timeout=timeout,
+            container=names["web"],
+            redactions=redactions,
+        )
+        _run(
+            ("docker", "exec", names["web"], "sh", "-c", "test \"$(id -u)\" = 101"),
+            timeout=30,
+        )
+        record("web_non_root_readiness", began)
+
+        began = time.monotonic()
+        _run(
+            (
+                "docker",
+                "run",
+                "--detach",
+                "--name",
+                names["worker"],
+                *common_runtime,
+                *_env_arguments(
+                    environment,
+                    role="worker",
+                    node_id=f"runtime-smoke-{slug}-worker",
+                ),
+                api_image,
+                "python",
+                "-m",
+                "celery",
+                "-A",
+                "govoplan_core.celery_app:celery",
+                "worker",
+                "--queues",
+                "default",
+                "--pool",
+                "solo",
+                "--concurrency",
+                "1",
+                "--hostname",
+                f"runtime-smoke-{slug}@%h",
+                "--loglevel",
+                "WARNING",
+            ),
+            timeout=timeout,
+            redactions=redactions,
+        )
+        _wait_for(
+            "GovOPlaN worker",
+            lambda: _run(
+                (
+                    "docker",
+                    "exec",
+                    names["api"],
+                    "python",
+                    "-c",
+                    (
+                        "from govoplan_core.celery_app import celery;"
+                        "result=celery.send_task('govoplan.ping',queue='default');"
+                        "assert result.get(timeout=10)=='pong'"
+                    ),
+                ),
+                check=False,
+                timeout=30,
+            ),
+            timeout=timeout,
+            container=names["worker"],
+            redactions=redactions,
+        )
+        _run(("docker", "stop", "--time", "20", names["worker"]), timeout=30)
+        record("worker_delivery_and_shutdown", began)
+    except SmokeError as exc:
+        for role in ("api", "web", "worker", "postgres", "redis"):
+            result = _run(
+                ("docker", "logs", "--tail", "100", names[role]),
+                check=False,
+                timeout=30,
+            )
+            if result.stdout or result.stderr:
+                detail = _tail(result.stdout + result.stderr, limit=8000)
+                for secret in redactions:
+                    detail = detail.replace(secret, "[redacted]")
+                print(f"--- {role} logs ---\n{detail}")
+        raise exc
+    finally:
+        for role in ("worker", "web", "api", "redis", "postgres"):
+            _run(
+                ("docker", "rm", "--force", names[role]),
+                check=False,
+                timeout=30,
+            )
+        _run(("docker", "volume", "rm", "--force", names["volume"]), check=False)
+        _run(("docker", "network", "rm", names["network"]), check=False)
+
+    return {
+        "schema_version": "1",
+        "evidence_kind": "govoplan.runtime-image-smoke",
+        "captured_at": _utc_now(),
+        "platform": platform,
+        "images": {
+            "api": api_image,
+            "web": web_image,
+            "postgres": postgres_image,
+            "redis": redis_image,
+        },
+        "result": {"state": "passed"},
+        "checks": checks,
+        "duration_seconds": round(time.monotonic() - started, 3),
+    }
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    try:
+        api_image = platform_image(args.api_metadata, args.platform, "API")
+        web_image = platform_image(args.web_metadata, args.platform, "Web")
+        postgres_image = platform_image(
+            args.postgres_metadata,
+            args.platform,
+            "PostgreSQL",
+        )
+        redis_image = platform_image(args.redis_metadata, args.platform, "Redis")
+        evidence = run_smoke(
+            api_image=api_image,
+            web_image=web_image,
+            postgres_image=postgres_image,
+            redis_image=redis_image,
+            platform=args.platform,
+            timeout=args.timeout_seconds,
+        )
+    except (OSError, SmokeError, ValueError) as exc:
+        print(f"runtime image smoke failed: {exc}")
+        return 1
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    temporary = args.output.with_suffix(args.output.suffix + ".tmp")
+    temporary.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    temporary.chmod(0o644)
+    temporary.replace(args.output)
+    print(f"Runtime image smoke evidence written to {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/deployment/build-deployer-zipapp.py

@@ -6,11 +6,13 @@ from __future__ import annotations
 import argparse
 from hashlib import sha256
 from pathlib import Path
-import zipapp
+from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
 
 
 ROOT = Path(__file__).resolve().parent
 DEFAULT_OUTPUT = ROOT.parent.parent / "runtime" / "deployment" / "govoplan-deploy.pyz"
+ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0)
+PYTHON_FILE_MODE = 0o100644
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -25,13 +27,7 @@ def main(argv: list[str] | None = None) -> int:
     temporary = output.with_name(f".{output.name}.tmp")
     if temporary.exists():
         temporary.unlink()
-    zipapp.create_archive(
-        ROOT,
-        target=temporary,
-        interpreter="/usr/bin/env python3",
-        compressed=True,
-        filter=_include_source,
-    )
+    _write_reproducible_zipapp(temporary)
     temporary.chmod(0o755)
     temporary.replace(output)
     digest = sha256(output.read_bytes()).hexdigest()
@@ -39,6 +35,42 @@ def main(argv: list[str] | None = None) -> int:
     return 0
 
 
+def _write_reproducible_zipapp(target: Path) -> None:
+    sources = tuple(
+        path
+        for path in sorted(ROOT.rglob("*"), key=lambda item: item.as_posix())
+        if path.is_file() and _include_source(path.relative_to(ROOT))
+    )
+    if not any(path.relative_to(ROOT).as_posix() == "__main__.py" for path in sources):
+        raise ValueError("deployment source has no __main__.py")
+    for path in sources:
+        if path.is_symlink():
+            raise ValueError(f"deployment source must not contain symlinks: {path}")
+
+    with target.open("wb") as handle:
+        handle.write(b"#!/usr/bin/env python3\n")
+        with ZipFile(
+            handle,
+            mode="w",
+            compression=ZIP_DEFLATED,
+            compresslevel=9,
+            strict_timestamps=True,
+        ) as archive:
+            for source in sources:
+                relative = source.relative_to(ROOT).as_posix()
+                info = ZipInfo(relative, date_time=ZIP_TIMESTAMP)
+                info.compress_type = ZIP_DEFLATED
+                info.create_system = 3
+                info.external_attr = PYTHON_FILE_MODE << 16
+                info.flag_bits |= 0x800
+                archive.writestr(
+                    info,
+                    source.read_bytes(),
+                    compress_type=ZIP_DEFLATED,
+                    compresslevel=9,
+                )
+
+
 def _include_source(path: Path) -> bool:
     return (
         "__pycache__" not in path.parts

tools/deployment/govoplan_deploy/backup_evidence.py

@@ -0,0 +1,500 @@
+"""Signed, provider-neutral backup and isolated-restore evidence."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+import re
+from typing import Any, Mapping
+from urllib.parse import urlsplit
+
+from .distribution import (
+    DistributionError,
+    load_bounded_json,
+    verify_signed_document,
+)
+
+
+MAX_BACKUP_EVIDENCE_BYTES = 1024 * 1024
+MAX_BACKUP_KEYRING_BYTES = 1024 * 1024
+DEFAULT_MAX_BACKUP_AGE_SECONDS = 24 * 60 * 60
+MAX_COORDINATION_SKEW_SECONDS = 5 * 60
+SHA256 = re.compile(r"^[0-9a-f]{64}$")
+TOKEN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
+REFERENCE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:[^\s]{1,2040}$")
+
+
+def load_backup_evidence(path: Path) -> dict[str, Any]:
+    return load_bounded_json(path, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES)
+
+
+def load_backup_keyring(path: Path) -> dict[str, Any]:
+    return load_bounded_json(path, maximum_bytes=MAX_BACKUP_KEYRING_BYTES)
+
+
+def verify_backup_evidence(
+    payload: Mapping[str, Any],
+    keyring: Mapping[str, Any],
+    *,
+    installation_id: str,
+    profile: str,
+    release: Mapping[str, object],
+    now: datetime | None = None,
+    max_age_seconds: int = DEFAULT_MAX_BACKUP_AGE_SECONDS,
+    openssl: str = "openssl",
+) -> dict[str, object]:
+    current = (now or datetime.now(UTC)).astimezone(UTC)
+    values = _validate_payload(payload, now=current, max_age_seconds=max_age_seconds)
+    if payload.get("installation_id") != installation_id:
+        raise DistributionError("backup evidence belongs to another installation")
+    subject = _object(payload.get("deployment_subject"), "deployment_subject")
+    if subject.get("profile") != profile:
+        raise DistributionError("backup evidence belongs to another deployment profile")
+    evidence_release = _object(payload.get("release"), "release")
+    for field in (
+        "channel",
+        "version",
+        "manifest_sha256",
+        "composition_sha256",
+        "api_image",
+        "web_image",
+    ):
+        if evidence_release.get(field) != release.get(field):
+            raise DistributionError(
+                f"backup evidence does not match release field {field!r}"
+            )
+    key_id = verify_signed_document(
+        payload,
+        keyring,
+        purpose="govoplan-backup-evidence",
+        label="backup evidence",
+        now=current,
+        openssl=openssl,
+    )
+    recovery_point = _object(payload.get("recovery_point"), "recovery_point")
+    restore = _object(payload.get("restore_drill"), "restore_drill")
+    return {
+        "evidence_id": payload["evidence_id"],
+        "recovery_point_id": recovery_point["id"],
+        "captured_at": recovery_point["captured_at"],
+        "expires_at": payload["expires_at"],
+        "restore_drill_id": restore["drill_id"],
+        "restore_started_at": restore["started_at"],
+        "restore_completed_at": restore["completed_at"],
+        "measured_rpo_seconds": restore["measured_rpo_seconds"],
+        "measured_rto_seconds": restore["measured_rto_seconds"],
+        "signature_key_id": key_id,
+        "component_count": values["component_count"],
+    }
+
+
+def _validate_payload(
+    payload: Mapping[str, Any],
+    *,
+    now: datetime,
+    max_age_seconds: int,
+) -> dict[str, int]:
+    if max_age_seconds < 60 or max_age_seconds > 30 * 24 * 60 * 60:
+        raise DistributionError("backup maximum age is out of bounds")
+    _exact_keys(
+        payload,
+        {
+            "schema_version",
+            "evidence_id",
+            "installation_id",
+            "deployment_subject",
+            "release",
+            "recovery_point",
+            "components",
+            "restore_drill",
+            "issued_at",
+            "expires_at",
+            "revoked",
+            "signatures",
+        },
+        "backup evidence",
+    )
+    if payload.get("schema_version") != "1":
+        raise DistributionError("unsupported backup evidence schema_version")
+    _token(payload.get("evidence_id"), "evidence_id")
+    _token(payload.get("installation_id"), "installation_id")
+    issued = _timestamp(payload.get("issued_at"), "issued_at")
+    expires = _timestamp(payload.get("expires_at"), "expires_at")
+    if issued > now or expires <= issued or expires <= now:
+        raise DistributionError("backup evidence is not currently valid")
+    if payload.get("revoked") is not False:
+        raise DistributionError("backup evidence is revoked")
+
+    subject = _object(payload.get("deployment_subject"), "deployment_subject")
+    _exact_keys(subject, {"profile", "topology", "subject_ref"}, "deployment_subject")
+    if subject.get("profile") not in {"evaluation", "self-hosted"}:
+        raise DistributionError("deployment_subject.profile is invalid")
+    _token(subject.get("topology"), "deployment_subject.topology")
+    _reference(subject.get("subject_ref"), "deployment_subject.subject_ref")
+
+    release = _object(payload.get("release"), "release")
+    _exact_keys(
+        release,
+        {
+            "channel",
+            "version",
+            "manifest_sha256",
+            "composition_sha256",
+            "api_image",
+            "web_image",
+        },
+        "release",
+    )
+    _token(release.get("channel"), "release.channel")
+    _token(release.get("version"), "release.version")
+    _sha256(release.get("manifest_sha256"), "release.manifest_sha256")
+    _sha256(release.get("composition_sha256"), "release.composition_sha256")
+    _image(release.get("api_image"), "release.api_image")
+    _image(release.get("web_image"), "release.web_image")
+
+    recovery = _object(payload.get("recovery_point"), "recovery_point")
+    _exact_keys(
+        recovery,
+        {"id", "captured_at", "consistency", "rpo_seconds", "write_fence"},
+        "recovery_point",
+    )
+    recovery_id = _token(recovery.get("id"), "recovery_point.id")
+    captured = _timestamp(recovery.get("captured_at"), "recovery_point.captured_at")
+    age = (now - captured).total_seconds()
+    if age < 0 or age > max_age_seconds:
+        raise DistributionError("backup recovery point is stale or in the future")
+    if recovery.get("consistency") not in {
+        "provider-atomic",
+        "application-quiesced",
+        "transaction-consistent",
+    }:
+        raise DistributionError("recovery_point.consistency is invalid")
+    declared_rpo = _bounded_integer(
+        recovery.get("rpo_seconds"),
+        "recovery_point.rpo_seconds",
+        maximum=30 * 24 * 60 * 60,
+    )
+    fence = _object(recovery.get("write_fence"), "recovery_point.write_fence")
+    _exact_keys(
+        fence,
+        {"mode", "token_sha256", "established_at"},
+        "recovery_point.write_fence",
+    )
+    if fence.get("mode") not in {
+        "provider-snapshot",
+        "application-quiesce",
+        "transaction-boundary",
+    }:
+        raise DistributionError("recovery_point.write_fence.mode is invalid")
+    _sha256(fence.get("token_sha256"), "recovery_point.write_fence.token_sha256")
+    established = _timestamp(
+        fence.get("established_at"),
+        "recovery_point.write_fence.established_at",
+    )
+    if abs((captured - established).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS:
+        raise DistributionError(
+            "backup write fence is not coordinated with recovery point"
+        )
+
+    components = _object(payload.get("components"), "components")
+    _exact_keys(
+        components,
+        {"database", "objects", "configuration", "key_custody"},
+        "components",
+    )
+    captured_components = [
+        _database_component(components.get("database")),
+        _objects_component(components.get("objects")),
+        _configuration_component(components.get("configuration")),
+        _key_custody_component(components.get("key_custody")),
+    ]
+    if any(
+        abs((component_time - captured).total_seconds()) > MAX_COORDINATION_SKEW_SECONDS
+        for component_time in captured_components
+    ):
+        raise DistributionError("backup components do not share one recovery point")
+
+    restore = _object(payload.get("restore_drill"), "restore_drill")
+    _exact_keys(
+        restore,
+        {
+            "drill_id",
+            "recovery_point_id",
+            "started_at",
+            "completed_at",
+            "isolated_target_ref",
+            "release_manifest_sha256",
+            "migration_heads_sha256",
+            "representative_object_manifest_sha256",
+            "database_verified",
+            "objects_verified",
+            "configuration_verified",
+            "key_custody_verified",
+            "semantic_checks",
+            "measured_rpo_seconds",
+            "measured_rto_seconds",
+            "evidence_ref",
+        },
+        "restore_drill",
+    )
+    _token(restore.get("drill_id"), "restore_drill.drill_id")
+    if restore.get("recovery_point_id") != recovery_id:
+        raise DistributionError("restore drill used another recovery point")
+    started = _timestamp(restore.get("started_at"), "restore_drill.started_at")
+    completed = _timestamp(restore.get("completed_at"), "restore_drill.completed_at")
+    if started < captured or completed < started or completed > issued:
+        raise DistributionError(
+            "restore drill completion is outside evidence chronology"
+        )
+    _reference(restore.get("isolated_target_ref"), "restore_drill.isolated_target_ref")
+    _reference(restore.get("evidence_ref"), "restore_drill.evidence_ref")
+    for field in (
+        "release_manifest_sha256",
+        "migration_heads_sha256",
+        "representative_object_manifest_sha256",
+    ):
+        _sha256(restore.get(field), f"restore_drill.{field}")
+    if restore.get("release_manifest_sha256") != release.get("manifest_sha256"):
+        raise DistributionError("restore drill used another immutable release")
+    for field in (
+        "database_verified",
+        "objects_verified",
+        "configuration_verified",
+        "key_custody_verified",
+    ):
+        if restore.get(field) is not True:
+            raise DistributionError(f"restore_drill.{field} must be true")
+    semantic = restore.get("semantic_checks")
+    if not isinstance(semantic, list) or not semantic or len(semantic) > 128:
+        raise DistributionError("restore_drill.semantic_checks must not be empty")
+    seen_checks: set[str] = set()
+    for index, raw in enumerate(semantic):
+        check = _object(raw, f"restore_drill.semantic_checks[{index}]")
+        _exact_keys(
+            check,
+            {"id", "status", "evidence_ref"},
+            f"restore_drill.semantic_checks[{index}]",
+        )
+        check_id = _token(check.get("id"), f"semantic_checks[{index}].id")
+        if check_id in seen_checks or check.get("status") != "passed":
+            raise DistributionError("restore drill semantic checks are invalid")
+        seen_checks.add(check_id)
+        _reference(check.get("evidence_ref"), f"semantic_checks[{index}].evidence_ref")
+    measured_rpo = _bounded_integer(
+        restore.get("measured_rpo_seconds"),
+        "restore_drill.measured_rpo_seconds",
+        maximum=30 * 24 * 60 * 60,
+    )
+    measured_rto = _bounded_integer(
+        restore.get("measured_rto_seconds"),
+        "restore_drill.measured_rto_seconds",
+        maximum=30 * 24 * 60 * 60,
+    )
+    if abs((completed - started).total_seconds() - measured_rto) > 5:
+        raise DistributionError("restore drill RTO does not match its timestamps")
+    if measured_rpo > declared_rpo:
+        raise DistributionError(
+            "restore drill exceeds the declared recovery point objective"
+        )
+    _validate_signatures(payload.get("signatures"))
+    return {"component_count": len(captured_components)}
+
+
+def _database_component(raw: object) -> datetime:
+    value = _object(raw, "components.database")
+    _exact_keys(
+        value,
+        {
+            "provider",
+            "artifact_ref",
+            "artifact_sha256",
+            "snapshot_id",
+            "lsn",
+            "protected",
+            "encryption_key_ref",
+            "captured_at",
+        },
+        "components.database",
+    )
+    _common_artifact(value, "components.database")
+    _token(value.get("snapshot_id"), "components.database.snapshot_id")
+    _bounded_text(value.get("lsn"), "components.database.lsn", maximum=256)
+    return _timestamp(value.get("captured_at"), "components.database.captured_at")
+
+
+def _objects_component(raw: object) -> datetime:
+    value = _object(raw, "components.objects")
+    _exact_keys(
+        value,
+        {
+            "provider",
+            "artifact_ref",
+            "manifest_sha256",
+            "version_id",
+            "object_count",
+            "total_bytes",
+            "protected",
+            "encryption_key_ref",
+            "captured_at",
+        },
+        "components.objects",
+    )
+    _token(value.get("provider"), "components.objects.provider")
+    _reference(value.get("artifact_ref"), "components.objects.artifact_ref")
+    _sha256(value.get("manifest_sha256"), "components.objects.manifest_sha256")
+    _token(value.get("version_id"), "components.objects.version_id")
+    _bounded_integer(value.get("object_count"), "components.objects.object_count")
+    _bounded_integer(value.get("total_bytes"), "components.objects.total_bytes")
+    _protected_key_reference(value, "components.objects")
+    return _timestamp(value.get("captured_at"), "components.objects.captured_at")
+
+
+def _configuration_component(raw: object) -> datetime:
+    value = _object(raw, "components.configuration")
+    _exact_keys(
+        value,
+        {
+            "artifact_ref",
+            "sha256",
+            "protected",
+            "encryption_key_ref",
+            "captured_at",
+        },
+        "components.configuration",
+    )
+    _reference(value.get("artifact_ref"), "components.configuration.artifact_ref")
+    _sha256(value.get("sha256"), "components.configuration.sha256")
+    _protected_key_reference(value, "components.configuration")
+    return _timestamp(value.get("captured_at"), "components.configuration.captured_at")
+
+
+def _key_custody_component(raw: object) -> datetime:
+    value = _object(raw, "components.key_custody")
+    _exact_keys(
+        value,
+        {"provider", "keyset_ref", "keyset_version", "recoverable", "captured_at"},
+        "components.key_custody",
+    )
+    _token(value.get("provider"), "components.key_custody.provider")
+    _reference(value.get("keyset_ref"), "components.key_custody.keyset_ref")
+    _token(value.get("keyset_version"), "components.key_custody.keyset_version")
+    if value.get("recoverable") is not True:
+        raise DistributionError("components.key_custody.recoverable must be true")
+    return _timestamp(value.get("captured_at"), "components.key_custody.captured_at")
+
+
+def _common_artifact(value: Mapping[str, Any], label: str) -> None:
+    _token(value.get("provider"), f"{label}.provider")
+    _reference(value.get("artifact_ref"), f"{label}.artifact_ref")
+    _sha256(value.get("artifact_sha256"), f"{label}.artifact_sha256")
+    _protected_key_reference(value, label)
+
+
+def _protected_key_reference(value: Mapping[str, Any], label: str) -> None:
+    if value.get("protected") is not True:
+        raise DistributionError(f"{label}.protected must be true")
+    _reference(value.get("encryption_key_ref"), f"{label}.encryption_key_ref")
+
+
+def _validate_signatures(raw: object) -> None:
+    if not isinstance(raw, list) or not raw or len(raw) > 16:
+        raise DistributionError("backup evidence signatures must not be empty")
+    seen: set[str] = set()
+    for index, item in enumerate(raw):
+        signature = _object(item, f"signatures[{index}]")
+        _exact_keys(signature, {"key_id", "algorithm", "value"}, f"signatures[{index}]")
+        key_id = _token(signature.get("key_id"), f"signatures[{index}].key_id")
+        if key_id in seen or signature.get("algorithm") != "ed25519":
+            raise DistributionError("backup evidence signatures are invalid")
+        seen.add(key_id)
+        encoded = signature.get("value")
+        if not isinstance(encoded, str) or len(encoded) > 256:
+            raise DistributionError("backup evidence signature value is invalid")
+
+
+def _reference(raw: object, label: str) -> str:
+    value = _bounded_text(raw, label, maximum=2048)
+    if REFERENCE.fullmatch(value) is None or "BEGIN " in value.upper():
+        raise DistributionError(f"{label} must be an opaque provider reference")
+    parsed = urlsplit(value)
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise DistributionError(f"{label} must not contain credentials or query data")
+    return value
+
+
+def _object(raw: object, label: str) -> dict[str, Any]:
+    if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw):
+        raise DistributionError(f"{label} must be an object")
+    return raw
+
+
+def _exact_keys(value: Mapping[str, Any], keys: set[str], label: str) -> None:
+    if set(value) != keys:
+        missing = sorted(keys - set(value))
+        extra = sorted(set(value) - keys)
+        detail = []
+        if missing:
+            detail.append("missing " + ", ".join(missing))
+        if extra:
+            detail.append("unknown " + ", ".join(extra))
+        raise DistributionError(f"{label} has invalid fields: {'; '.join(detail)}")
+
+
+def _timestamp(raw: object, label: str) -> datetime:
+    value = _bounded_text(raw, label, maximum=64)
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise DistributionError(f"{label} must be an RFC3339 timestamp") from exc
+    if parsed.tzinfo is None:
+        raise DistributionError(f"{label} must include a timezone")
+    return parsed.astimezone(UTC)
+
+
+def _token(raw: object, label: str) -> str:
+    value = _bounded_text(raw, label, maximum=128)
+    if TOKEN.fullmatch(value) is None:
+        raise DistributionError(f"{label} is invalid")
+    return value
+
+
+def _sha256(raw: object, label: str) -> str:
+    value = _bounded_text(raw, label, maximum=64)
+    if SHA256.fullmatch(value) is None:
+        raise DistributionError(f"{label} must be a lowercase SHA-256 digest")
+    return value
+
+
+def _image(raw: object, label: str) -> str:
+    value = _bounded_text(raw, label, maximum=300)
+    if IMAGE.fullmatch(value) is None:
+        raise DistributionError(f"{label} must be an OCI image pinned by sha256")
+    return value
+
+
+def _bounded_text(raw: object, label: str, *, maximum: int) -> str:
+    if not isinstance(raw, str) or not raw or len(raw) > maximum or "\n" in raw:
+        raise DistributionError(f"{label} is invalid")
+    return raw
+
+
+def _bounded_integer(
+    raw: object,
+    label: str,
+    *,
+    maximum: int = 2**63 - 1,
+) -> int:
+    if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0 or raw > maximum:
+        raise DistributionError(f"{label} is out of bounds")
+    return raw
+
+
+__all__ = [
+    "DEFAULT_MAX_BACKUP_AGE_SECONDS",
+    "MAX_BACKUP_EVIDENCE_BYTES",
+    "MAX_BACKUP_KEYRING_BYTES",
+    "load_backup_evidence",
+    "load_backup_keyring",
+    "verify_backup_evidence",
+]

tools/deployment/govoplan_deploy/bundle.py

@@ -28,7 +28,26 @@ PLAN_FILENAME = "plan.json"
 RECEIPT_FILENAME = "receipt.json"
 MANIFEST_FILENAME = "distribution-manifest.json"
 KEYRING_FILENAME = "distribution-keyring.json"
+BACKUP_EVIDENCE_FILENAME = "backup-evidence.json"
+BACKUP_KEYRING_FILENAME = "backup-keyring.json"
+BACKUP_VERIFICATION_FILENAME = "backup-verification.json"
 LOCK_FILENAME = ".deployment.lock"
+BACKUP_RUNTIME_ENV_KEYS = (
+    "GOVOPLAN_BACKUP_EVIDENCE_STATE",
+    "GOVOPLAN_BACKUP_EVIDENCE_ID",
+    "GOVOPLAN_BACKUP_RECOVERY_POINT_ID",
+    "GOVOPLAN_BACKUP_RESTORE_DRILL_ID",
+    "GOVOPLAN_BACKUP_EVIDENCE_SHA256",
+    "GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256",
+    "GOVOPLAN_BACKUP_CAPTURED_AT",
+    "GOVOPLAN_BACKUP_EXPIRES_AT",
+    "GOVOPLAN_BACKUP_RESTORE_STARTED_AT",
+    "GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT",
+    "GOVOPLAN_BACKUP_VERIFIED_AT",
+    "GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS",
+    "GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS",
+    "GOVOPLAN_BACKUP_COMPONENT_COUNT",
+)
 RUNTIME_ENV_KEYS = (
     "APP_ENV",
     "GOVOPLAN_INSTALL_PROFILE",
@@ -78,6 +97,7 @@ RUNTIME_ENV_KEYS = (
     "FILE_STORAGE_S3_BUCKET",
     "FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
     "FILE_STORAGE_S3_ENDPOINT_TRUSTED",
+    *BACKUP_RUNTIME_ENV_KEYS,
 )
 
 
@@ -95,6 +115,9 @@ class BundlePaths:
     receipt: Path
     manifest: Path
     keyring: Path
+    backup_evidence: Path
+    backup_keyring: Path
+    backup_verification: Path
     lock: Path
 
 
@@ -116,6 +139,9 @@ def bundle_paths(root: Path) -> BundlePaths:
         receipt=resolved / RECEIPT_FILENAME,
         manifest=resolved / MANIFEST_FILENAME,
         keyring=resolved / KEYRING_FILENAME,
+        backup_evidence=resolved / BACKUP_EVIDENCE_FILENAME,
+        backup_keyring=resolved / BACKUP_KEYRING_FILENAME,
+        backup_verification=resolved / BACKUP_VERIFICATION_FILENAME,
         lock=resolved / LOCK_FILENAME,
     )
 
@@ -601,6 +627,7 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
             "tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"],
             "security_opt": ["no-new-privileges:true"],
             "cap_drop": ["ALL"],
+            "cap_add": ["NET_BIND_SERVICE"],
             "volumes": [
                 f"./{CADDY_CONFIG_FILENAME}:/etc/caddy/Caddyfile:ro",
                 "caddy-data:/data",

tools/deployment/govoplan_deploy/cli.py

@@ -20,7 +20,14 @@ from typing import Iterator, Mapping, Sequence
 from urllib.error import URLError
 from urllib.request import urlopen
 
+from .backup_evidence import (
+    DEFAULT_MAX_BACKUP_AGE_SECONDS,
+    MAX_BACKUP_EVIDENCE_BYTES,
+    MAX_BACKUP_KEYRING_BYTES,
+    verify_backup_evidence,
+)
 from .bundle import (
+    BACKUP_RUNTIME_ENV_KEYS,
     atomic_write,
     bundle_paths,
     canonical_json,
@@ -72,7 +79,12 @@ from .kubernetes import (
     render_kubernetes,
     write_secret_creation_hint,
 )
-from .planning import DeploymentPlan, build_plan
+from .planning import (
+    DeploymentPlan,
+    build_plan,
+    release_change_requires_backup,
+    verify_stored_backup_evidence,
+)
 from .recovery import (
     DeploymentOperationJournal,
     list_operations,
@@ -183,6 +195,22 @@ def build_parser() -> argparse.ArgumentParser:
         help="Load verified archives into Docker using fixed image-load commands.",
     )
 
+    verify_backup = subparsers.add_parser(
+        "verify-backup",
+        help="Verify and optionally adopt signed coordinated backup evidence.",
+    )
+    _directory_argument(verify_backup)
+    evidence_source = verify_backup.add_mutually_exclusive_group(required=True)
+    evidence_source.add_argument("--evidence", type=Path)
+    evidence_source.add_argument("--evidence-url")
+    verify_backup.add_argument("--evidence-sha256", required=True)
+    verify_backup.add_argument("--trusted-keyring", type=Path, required=True)
+    verify_backup.add_argument(
+        "--allow-private-evidence-host",
+        action="store_true",
+    )
+    verify_backup.add_argument("--adopt", action="store_true")
+
     kubernetes = subparsers.add_parser(
         "render-kubernetes",
         help="Export the stateless multi-host runtime for Kubernetes.",
@@ -392,6 +420,8 @@ def main(argv: Sequence[str] | None = None) -> int:
             return _verify_release(args)
         if args.command == "verify-offline-images":
             return _verify_offline_images(args)
+        if args.command == "verify-backup":
+            return _verify_backup(args)
         if args.command == "render-kubernetes":
             return _render_kubernetes(args)
         if args.command == "verify-kubernetes":
@@ -534,15 +564,17 @@ def _render_or_doctor(args: argparse.Namespace) -> int:
 
 def _apply(args: argparse.Namespace) -> int:
     paths = bundle_paths(args.directory)
-    spec = load_spec(paths.spec)
-    if args.allow_unverified_images and spec.profile != "evaluation":
-        raise ValueError(
-            "--allow-unverified-images is restricted to evaluation installations"
-        )
     ensure_private_directory(paths.root)
     with _deployment_lock(paths.lock):
+        spec = load_spec(paths.spec)
+        previous_receipt = _read_json_object(paths.receipt)
+        backup_required = release_change_requires_backup(spec, previous_receipt)
+        if args.allow_unverified_images and spec.profile != "evaluation":
+            raise ValueError(
+                "--allow-unverified-images is restricted to evaluation installations"
+            )
         secrets = reconcile_runtime_environment(spec, read_env(paths.env))
-        _write_bundle(spec, paths, secrets)
+        secrets = _write_bundle(spec, paths, secrets)
         plan = build_plan(spec, paths, include_host_checks=True)
         _write_plan(paths.plan, plan)
         effective_errors = [
@@ -577,6 +609,36 @@ def _apply(args: argparse.Namespace) -> int:
             paths,
             plan=plan.to_dict(),
         )
+        try:
+            if backup_required:
+                backup_summary = verify_stored_backup_evidence(
+                    spec,
+                    paths,
+                    receipt=previous_receipt,
+                )
+                journal.record(
+                    "backup-evidence-verified",
+                    "succeeded",
+                    dict(backup_summary),
+                )
+            else:
+                journal.record(
+                    "backup-evidence-not-required",
+                    "succeeded",
+                    {"release_change": False},
+                )
+        except BaseException as exc:
+            journal.record(
+                "backup-evidence-rejected",
+                "blocked",
+                {
+                    "phase": "preflight",
+                    "exception_type": type(exc).__name__,
+                    "migration_started": False,
+                },
+            )
+            journal.failed(exc)
+            raise
         compose = [
             docker,
             "compose",
@@ -624,6 +686,29 @@ def _apply(args: argparse.Namespace) -> int:
                 "succeeded",
                 {"services": mutable_runtime_services, "timeout_seconds": 120},
             )
+            if backup_required:
+                try:
+                    backup_summary = verify_stored_backup_evidence(
+                        spec,
+                        paths,
+                        receipt=previous_receipt,
+                    )
+                except BaseException as exc:
+                    journal.record(
+                        "backup-evidence-rejected",
+                        "blocked",
+                        {
+                            "phase": "migration-boundary",
+                            "exception_type": type(exc).__name__,
+                            "migration_started": False,
+                        },
+                    )
+                    raise
+                journal.record(
+                    "backup-evidence-reverified",
+                    "succeeded",
+                    dict(backup_summary),
+                )
             journal.migration_started()
             _run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
             journal.migration_completed()
@@ -898,6 +983,108 @@ def _verify_offline_images(args: argparse.Namespace) -> int:
     return 0
 
 
+def _verify_backup(args: argparse.Namespace) -> int:
+    paths = bundle_paths(args.directory)
+    ensure_private_directory(paths.root)
+    with _deployment_lock(paths.lock):
+        return _verify_backup_locked(args, paths)
+
+
+def _verify_backup_locked(args: argparse.Namespace, paths) -> int:
+    spec = load_spec(paths.spec)
+    expected_digest = str(args.evidence_sha256 or "").strip().lower()
+    if len(expected_digest) != 64 or any(
+        character not in "0123456789abcdef" for character in expected_digest
+    ):
+        raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest")
+    if args.evidence_url:
+        encoded_evidence = fetch_bounded_https(
+            str(args.evidence_url),
+            maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
+            allow_private_host=args.allow_private_evidence_host,
+        )
+        evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
+    else:
+        evidence_path = args.evidence.expanduser().resolve()
+        encoded_evidence = read_bounded_bytes(
+            evidence_path,
+            maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
+        )
+        evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
+    if encoded_evidence != canonical_distribution_json(evidence):
+        raise DistributionError("backup evidence is not canonical JSON")
+    if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest:
+        raise DistributionError("backup evidence SHA-256 does not match")
+    keyring_path = args.trusted_keyring.expanduser().resolve()
+    keyring = load_bounded_json(
+        keyring_path,
+        maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
+    )
+    encoded_keyring = canonical_distribution_json(keyring)
+    receipt = _read_json_object(paths.receipt)
+    previous_release = receipt.get("release") if receipt else None
+    release: Mapping[str, object] = (
+        previous_release
+        if isinstance(previous_release, Mapping)
+        else {
+            "channel": spec.release.channel,
+            "version": spec.release.version,
+            "manifest_sha256": spec.release.manifest_sha256,
+            "composition_sha256": spec.release.composition_sha256,
+            "api_image": spec.release.api_image,
+            "web_image": spec.release.web_image,
+        }
+    )
+    summary = verify_backup_evidence(
+        evidence,
+        keyring,
+        installation_id=spec.installation_id,
+        profile=spec.profile,
+        release=release,
+        max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS,
+    )
+    print(
+        "Verified coordinated recovery point "
+        f"{summary['recovery_point_id']} with restore drill "
+        f"{summary['restore_drill_id']} and trusted key "
+        f"{summary['signature_key_id']}."
+    )
+    if not args.adopt:
+        return 0
+    verification = {
+        "schema_version": 1,
+        "evidence_sha256": expected_digest,
+        "keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(),
+        "signature_key_id": summary["signature_key_id"],
+        "verified_at": _now(),
+        "evidence_id": summary["evidence_id"],
+        "recovery_point_id": summary["recovery_point_id"],
+        "restore_drill_id": summary["restore_drill_id"],
+        "release_manifest_sha256": release.get("manifest_sha256"),
+        "captured_at": summary["captured_at"],
+        "expires_at": summary["expires_at"],
+        "restore_started_at": summary["restore_started_at"],
+        "restore_completed_at": summary["restore_completed_at"],
+        "measured_rpo_seconds": summary["measured_rpo_seconds"],
+        "measured_rto_seconds": summary["measured_rto_seconds"],
+        "component_count": summary["component_count"],
+    }
+    atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600)
+    atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600)
+    atomic_write(
+        paths.backup_verification,
+        canonical_json(verification),
+        mode=0o600,
+    )
+    _write_bundle(
+        spec,
+        paths,
+        reconcile_runtime_environment(spec, read_env(paths.env)),
+    )
+    print(f"Adopted signed backup evidence in {paths.root}.")
+    return 0
+
+
 def _selected_dependency_images(
     spec: InstallationSpec,
     *,
@@ -929,6 +1116,30 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
     paths = bundle_paths(args.directory)
     spec = load_spec(paths.spec)
     environment = reconcile_runtime_environment(spec, read_env(paths.env))
+    environment.update(_backup_runtime_environment(spec, paths))
+    receipt = _read_json_object(paths.receipt)
+    backup_required = release_change_requires_backup(spec, receipt)
+    backup_summary: Mapping[str, object] | None = None
+    evidence_files = (
+        paths.backup_evidence,
+        paths.backup_keyring,
+        paths.backup_verification,
+    )
+    if backup_required:
+        backup_summary = verify_stored_backup_evidence(
+            spec,
+            paths,
+            receipt=receipt,
+        )
+    elif all(path.is_file() for path in evidence_files):
+        try:
+            backup_summary = verify_stored_backup_evidence(
+                spec,
+                paths,
+                receipt=receipt,
+            )
+        except (DistributionError, OSError):
+            backup_summary = None
     manifest = render_kubernetes(
         spec,
         environment,
@@ -936,6 +1147,8 @@ def _render_kubernetes(args: argparse.Namespace) -> int:
         secret_name=args.secret_name,
         tls_secret_name=args.tls_secret_name,
         ingress_class_name=args.ingress_class_name,
+        backup_required=backup_required,
+        backup_evidence=backup_summary,
     )
     output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
     atomic_write(output, canonical_json(manifest), mode=0o600)
@@ -1027,6 +1240,7 @@ def _deployment_receipt(
     return {
         "schema_version": 1,
         "installation_id": spec.installation_id,
+        "profile": spec.profile,
         "applied_at": _now(),
         "spec_sha256": digest_json(spec.to_dict()),
         "compose_sha256": digest_json(render_compose(spec)),
@@ -1064,6 +1278,16 @@ def _deployment_receipt(
     }
 
 
+def _read_json_object(path: Path) -> dict[str, object]:
+    if not path.is_file():
+        return {}
+    try:
+        value = load_bounded_json(path, maximum_bytes=64 * 1024)
+    except (DistributionError, OSError):
+        return {}
+    return value
+
+
 def _updated_spec(
     current: InstallationSpec, args: argparse.Namespace
 ) -> InstallationSpec:
@@ -1222,10 +1446,12 @@ def _write_bundle(
     spec: InstallationSpec,
     paths,
     secrets: Mapping[str, str],
-) -> None:
+) -> dict[str, str]:
     ensure_private_directory(paths.root)
+    runtime_environment = dict(secrets)
+    runtime_environment.update(_backup_runtime_environment(spec, paths))
     atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
-    write_env(paths.env, secrets)
+    write_env(paths.env, runtime_environment)
     atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
     atomic_write(
         paths.load_balancer_config,
@@ -1247,6 +1473,62 @@ def _write_bundle(
         render_garage_config().encode("utf-8"),
         mode=0o644,
     )
+    return dict(sorted(runtime_environment.items()))
+
+
+def _backup_runtime_environment(
+    spec: InstallationSpec,
+    paths,
+) -> dict[str, str]:
+    values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS}
+    evidence_files = (
+        paths.backup_evidence,
+        paths.backup_keyring,
+        paths.backup_verification,
+    )
+    if not any(path.is_file() for path in evidence_files):
+        values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent"
+        return values
+    if not all(path.is_file() for path in evidence_files):
+        values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
+        return values
+    verification = _read_json_object(paths.backup_verification)
+    try:
+        summary = verify_stored_backup_evidence(
+            spec,
+            paths,
+            receipt=_read_json_object(paths.receipt),
+        )
+    except (DistributionError, OSError):
+        values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
+        return values
+    values.update(
+        {
+            "GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
+            "GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]),
+            "GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]),
+            "GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]),
+            "GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]),
+            "GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str(
+                verification["release_manifest_sha256"]
+            ),
+            "GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]),
+            "GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]),
+            "GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]),
+            "GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str(
+                summary["restore_completed_at"]
+            ),
+            "GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]),
+            "GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str(
+                summary["measured_rpo_seconds"]
+            ),
+            "GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str(
+                summary["measured_rto_seconds"]
+            ),
+            "GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]),
+        }
+    )
+    return values
 
 
 def _write_plan(path: Path, plan: DeploymentPlan) -> None:

tools/deployment/govoplan_deploy/distribution.py

@@ -74,7 +74,9 @@ def read_bounded_bytes(path: Path, *, maximum_bytes: int) -> bytes:
     try:
         opened = os.fstat(descriptor)
         if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
-            raise DistributionError(f"trusted JSON file is invalid or too large: {path}")
+            raise DistributionError(
+                f"trusted JSON file is invalid or too large: {path}"
+            )
         chunks: list[bytes] = []
         total = 0
         while True:
@@ -109,7 +111,9 @@ def fetch_bounded_https(
     if parsed.scheme != "https" or not parsed.hostname:
         raise DistributionError("distribution downloads require an absolute HTTPS URL")
     if parsed.username or parsed.password or parsed.fragment:
-        raise DistributionError("distribution URL must not contain credentials or a fragment")
+        raise DistributionError(
+            "distribution URL must not contain credentials or a fragment"
+        )
     if not allow_private_host:
         _require_public_host(parsed.hostname)
     request = Request(url, headers={"Accept": "application/json"})
@@ -172,9 +176,11 @@ def validate_manifest(
         raise DistributionError(
             f"distribution channel is {channel!r}, expected {expected_channel!r}"
         )
-    if isinstance(payload.get("sequence"), bool) or not isinstance(
-        payload.get("sequence"), int
-    ) or int(payload["sequence"]) < 1:
+    if (
+        isinstance(payload.get("sequence"), bool)
+        or not isinstance(payload.get("sequence"), int)
+        or int(payload["sequence"]) < 1
+    ):
         raise DistributionError("distribution sequence must be a positive integer")
     _token(payload.get("version"), "version", maximum=128, pattern=TOKEN)
     issued = _datetime(payload.get("issued_at"), "issued_at")
@@ -283,11 +289,41 @@ def verify_manifest(
 ) -> str:
     current = (now or datetime.now(UTC)).astimezone(UTC)
     validate_manifest(payload, expected_channel=expected_channel, now=current)
-    keys = _trusted_keys(keyring, now=current)
+    return verify_signed_document(
+        payload,
+        keyring,
+        purpose="govoplan-runtime-distribution",
+        label="distribution",
+        now=current,
+        openssl=openssl,
+    )
+
+
+def verify_signed_document(
+    payload: Mapping[str, Any],
+    keyring: Mapping[str, Any],
+    *,
+    purpose: str,
+    label: str,
+    now: datetime,
+    openssl: str = "openssl",
+) -> str:
+    keys = _trusted_keys(keyring, now=now, purpose=purpose, label=label)
     signed = canonical_signed_payload(payload)
     failures: list[str] = []
-    for item in payload["signatures"]:
+    signatures = payload.get("signatures")
+    if not isinstance(signatures, list) or not signatures:
+        raise DistributionError(f"{label} has no signatures")
+    for index, raw in enumerate(signatures):
+        item = _object(raw, f"{label}.signatures[{index}]")
+        _exact_keys(
+            item,
+            required={"key_id", "algorithm", "value"},
+            label=f"{label}.signatures[{index}]",
+        )
         key_id = str(item["key_id"])
+        if KEY_ID.fullmatch(key_id) is None or item.get("algorithm") != "ed25519":
+            raise DistributionError(f"{label} signature is invalid")
         public_key = keys.get(key_id)
         if public_key is None:
             continue
@@ -303,8 +339,10 @@ def verify_manifest(
             failures.append(f"{key_id}: {exc}")
             continue
         return key_id
-    detail = "; ".join(failures) if failures else "no signature used an active trusted key"
-    raise DistributionError(f"distribution signature verification failed: {detail}")
+    detail = (
+        "; ".join(failures) if failures else "no signature used an active trusted key"
+    )
+    raise DistributionError(f"{label} signature verification failed: {detail}")
 
 
 def verify_manifest_binding(
@@ -319,7 +357,9 @@ def verify_manifest_binding(
     dependencies: Mapping[str, str],
 ) -> None:
     if payload.get("channel") != channel or payload.get("version") != version:
-        raise DistributionError("stored manifest does not match release channel/version")
+        raise DistributionError(
+            "stored manifest does not match release channel/version"
+        )
     images = _object(payload.get("images"), "images")
     if _object(images.get("api"), "images.api").get("index") != api_image:
         raise DistributionError("stored manifest does not match API image")
@@ -372,9 +412,9 @@ def verify_offline_image_index(
         archive = root / archive_relative
         if reference in references:
             raise DistributionError("offline image index contains duplicate references")
-        if _sha256_regular_file(archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES) != _sha256(
-            value.get("sha256"), "offline image sha256"
-        ):
+        if _sha256_regular_file(
+            archive, maximum_bytes=MAX_OFFLINE_IMAGE_BYTES
+        ) != _sha256(value.get("sha256"), "offline image sha256"):
             raise DistributionError(f"offline image archive digest mismatch: {archive}")
         references[reference] = archive
     missing = sorted(set(expected_references) - set(references))
@@ -418,19 +458,21 @@ def _trusted_keys(
     keyring: Mapping[str, Any],
     *,
     now: datetime,
+    purpose: str,
+    label: str,
 ) -> dict[str, str]:
     _exact_keys(
         keyring,
         required={"schema_version", "purpose", "keys"},
-        label="distribution keyring",
+        label=f"{label} keyring",
     )
     if keyring.get("schema_version") != "1":
-        raise DistributionError("unsupported distribution keyring schema_version")
-    if keyring.get("purpose") != "govoplan-runtime-distribution":
-        raise DistributionError("distribution keyring has the wrong purpose")
+        raise DistributionError(f"unsupported {label} keyring schema_version")
+    if keyring.get("purpose") != purpose:
+        raise DistributionError(f"{label} keyring has the wrong purpose")
     values = keyring.get("keys")
     if not isinstance(values, list) or not values:
-        raise DistributionError("distribution keyring contains no keys")
+        raise DistributionError(f"{label} keyring contains no keys")
     trusted: dict[str, str] = {}
     for index, item in enumerate(values):
         key = _object(item, f"keyring.keys[{index}]")
@@ -453,11 +495,11 @@ def _trusted_keys(
             pattern=KEY_ID,
         )
         if key_id in trusted:
-            raise DistributionError("distribution keyring contains duplicate key ids")
+            raise DistributionError(f"{label} keyring contains duplicate key ids")
         if key.get("algorithm") != "ed25519":
-            raise DistributionError("distribution key must use ed25519")
+            raise DistributionError(f"{label} key must use ed25519")
         if key.get("status") not in {"active", "retired", "revoked"}:
-            raise DistributionError("distribution key has an invalid status")
+            raise DistributionError(f"{label} key has an invalid status")
         not_before = _datetime(key.get("not_before"), "key.not_before")
         expires = _datetime(key.get("expires_at"), "key.expires_at")
         public_key = key.get("public_key_pem")
@@ -466,11 +508,11 @@ def _trusted_keys(
             or len(public_key.encode("utf-8")) > 8192
             or "BEGIN PUBLIC KEY" not in public_key
         ):
-            raise DistributionError("distribution key has an invalid public key")
+            raise DistributionError(f"{label} key has an invalid public key")
         if key.get("status") == "active" and not_before <= now < expires:
             trusted[key_id] = public_key
     if not trusted:
-        raise DistributionError("distribution keyring has no currently active keys")
+        raise DistributionError(f"{label} keyring has no currently active keys")
     return trusted
 
 
@@ -534,13 +576,17 @@ def _require_public_host(hostname: str) -> None:
             for value in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM)
         }
     except OSError as exc:
-        raise DistributionError(f"distribution host cannot be resolved: {hostname}") from exc
+        raise DistributionError(
+            f"distribution host cannot be resolved: {hostname}"
+        ) from exc
     if not addresses:
         raise DistributionError("distribution host resolved to no addresses")
     for value in addresses:
         address = ipaddress.ip_address(value)
         if not address.is_global:
-            raise DistributionError("distribution host resolves to a non-public address")
+            raise DistributionError(
+                "distribution host resolves to a non-public address"
+            )
 
 
 def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
@@ -553,7 +599,9 @@ def _sha256_regular_file(path: Path, *, maximum_bytes: int) -> str:
     try:
         opened = os.fstat(descriptor)
         if not stat.S_ISREG(opened.st_mode) or opened.st_size > maximum_bytes:
-            raise DistributionError(f"immutable artifact is invalid or too large: {path}")
+            raise DistributionError(
+                f"immutable artifact is invalid or too large: {path}"
+            )
         while True:
             chunk = os.read(descriptor, 1024 * 1024)
             if not chunk:
@@ -602,7 +650,11 @@ def _token(
     maximum: int,
     pattern: re.Pattern[str],
 ) -> str:
-    if not isinstance(value, str) or len(value) > maximum or pattern.fullmatch(value) is None:
+    if (
+        not isinstance(value, str)
+        or len(value) > maximum
+        or pattern.fullmatch(value) is None
+    ):
         raise DistributionError(f"{label} is invalid")
     return value
 
@@ -626,7 +678,11 @@ def _sha256(value: object, label: str) -> str:
 
 
 def _digest_image(value: object, label: str) -> str:
-    if not isinstance(value, str) or len(value) > 300 or DIGEST_IMAGE.fullmatch(value) is None:
+    if (
+        not isinstance(value, str)
+        or len(value) > 300
+        or DIGEST_IMAGE.fullmatch(value) is None
+    ):
         raise DistributionError(f"{label} must be an OCI image pinned by sha256")
     return value
 
@@ -635,7 +691,12 @@ def _https_url(value: object, label: str) -> str:
     if not isinstance(value, str) or len(value) > 2048:
         raise DistributionError(f"{label} must be an HTTPS URL")
     parsed = urlsplit(value)
-    if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
+    if (
+        parsed.scheme != "https"
+        or not parsed.netloc
+        or parsed.username
+        or parsed.password
+    ):
         raise DistributionError(f"{label} must be an HTTPS URL without credentials")
     return value
 

tools/deployment/govoplan_deploy/kubernetes.py

@@ -10,6 +10,7 @@ import re
 from typing import Any, Mapping
 from urllib.parse import urlsplit
 
+from .bundle import BACKUP_RUNTIME_ENV_KEYS
 from .model import InstallationSpec, image_is_digest_pinned
 
 
@@ -55,6 +56,7 @@ _CONFIG_KEYS = (
     "FILE_STORAGE_S3_BUCKET",
     "FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
     "FILE_STORAGE_S3_ENDPOINT_TRUSTED",
+    *BACKUP_RUNTIME_ENV_KEYS,
 )
 _QUEUE_NAME = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$")
 
@@ -75,6 +77,8 @@ def render_kubernetes(
     secret_name: str = "govoplan-runtime",
     tls_secret_name: str = "govoplan-tls",
     ingress_class_name: str | None = None,
+    backup_required: bool = True,
+    backup_evidence: Mapping[str, object] | None = None,
 ) -> dict[str, Any]:
     """Render runtime roles only; shared state services stay externally managed."""
 
@@ -199,6 +203,8 @@ def render_kubernetes(
                 environment,
                 "MIGRATION",
             ),
+            backup_required=backup_required,
+            backup_evidence=backup_evidence,
         ),
     ]
     for pool in worker_pools:
@@ -765,9 +771,28 @@ def _migration_job(
     secret_name: str,
     service_account: str,
     database_environment: Mapping[str, str],
+    backup_required: bool,
+    backup_evidence: Mapping[str, object] | None,
 ) -> dict[str, Any]:
     job_labels = {**labels, "app.kubernetes.io/component": "migration"}
     job_name = _name_with_suffix(name, f"migrate-{release_key}")
+    backup_annotations = {
+        "govoplan.add-ideas.de/backup-required": str(backup_required).lower(),
+    }
+    if backup_evidence is not None:
+        backup_annotations.update(
+            {
+                "govoplan.add-ideas.de/backup-evidence-sha256": str(
+                    backup_evidence["evidence_sha256"]
+                ),
+                "govoplan.add-ideas.de/recovery-point": str(
+                    backup_evidence["recovery_point_id"]
+                ),
+                "govoplan.add-ideas.de/restore-drill": str(
+                    backup_evidence["restore_drill_id"]
+                ),
+            }
+        )
     return {
         "apiVersion": "batch/v1",
         "kind": "Job",
@@ -777,7 +802,7 @@ def _migration_job(
             "labels": job_labels,
             "annotations": {
                 "govoplan.add-ideas.de/recovery-mode": "forward-recovery",
-                "govoplan.add-ideas.de/backup-required": "true",
+                **backup_annotations,
                 "argocd.argoproj.io/sync-wave": "-1",
             },
         },

tools/deployment/govoplan_deploy/planning.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -18,6 +19,11 @@ from urllib.error import HTTPError, URLError
 from urllib.parse import urlsplit
 from urllib.request import Request, urlopen
 
+from .backup_evidence import (
+    MAX_BACKUP_EVIDENCE_BYTES,
+    MAX_BACKUP_KEYRING_BYTES,
+    verify_backup_evidence,
+)
 from .bundle import (
     BundlePaths,
     canonical_json,
@@ -33,8 +39,11 @@ from .distribution import (
     MAX_KEYRING_BYTES,
     MAX_MANIFEST_BYTES,
     DistributionError,
+    canonical_json as canonical_distribution_json,
+    decode_json_bytes,
     file_sha256,
     load_bounded_json,
+    read_bounded_bytes,
     verify_manifest,
     verify_manifest_binding,
 )
@@ -232,6 +241,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
             )
 
     checks.extend(_distribution_checks(spec, paths))
+    checks.extend(
+        _backup_evidence_checks(spec, paths, receipt=_read_receipt(paths.receipt))
+    )
 
     values = read_env(paths.env)
     required = {"MASTER_KEY_B64", "DATABASE_URL"}
@@ -350,6 +362,189 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
     return tuple(checks)
 
 
+def release_change_requires_backup(
+    spec: InstallationSpec,
+    receipt: Mapping[str, object],
+) -> bool:
+    if spec.profile != "self-hosted" or not receipt:
+        return False
+    previous = receipt.get("release")
+    if not isinstance(previous, Mapping):
+        return True
+    desired = {
+        "channel": spec.release.channel,
+        "version": spec.release.version,
+        "manifest_sha256": spec.release.manifest_sha256,
+        "composition_sha256": spec.release.composition_sha256,
+        "api_image": spec.release.api_image,
+        "web_image": spec.release.web_image,
+    }
+    return any(previous.get(key) != value for key, value in desired.items())
+
+
+def verify_stored_backup_evidence(
+    spec: InstallationSpec,
+    paths: BundlePaths,
+    *,
+    receipt: Mapping[str, object],
+) -> dict[str, object]:
+    verification = load_bounded_json(
+        paths.backup_verification,
+        maximum_bytes=64 * 1024,
+    )
+    expected_fields = {
+        "schema_version",
+        "evidence_sha256",
+        "keyring_sha256",
+        "signature_key_id",
+        "verified_at",
+        "evidence_id",
+        "recovery_point_id",
+        "restore_drill_id",
+        "release_manifest_sha256",
+        "captured_at",
+        "expires_at",
+        "restore_started_at",
+        "restore_completed_at",
+        "measured_rpo_seconds",
+        "measured_rto_seconds",
+        "component_count",
+    }
+    if set(verification) != expected_fields or verification.get("schema_version") != 1:
+        raise DistributionError("backup verification receipt is malformed")
+    encoded_evidence = read_bounded_bytes(
+        paths.backup_evidence,
+        maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
+    )
+    encoded_keyring = read_bounded_bytes(
+        paths.backup_keyring,
+        maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
+    )
+    evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
+    keyring = decode_json_bytes(encoded_keyring, label="backup keyring")
+    if encoded_evidence != canonical_distribution_json(evidence):
+        raise DistributionError("stored backup evidence is not canonical JSON")
+    if encoded_keyring != canonical_distribution_json(keyring):
+        raise DistributionError("stored backup keyring is not canonical JSON")
+    evidence_digest = hashlib.sha256(encoded_evidence).hexdigest()
+    keyring_digest = hashlib.sha256(encoded_keyring).hexdigest()
+    if evidence_digest != verification.get("evidence_sha256"):
+        raise DistributionError("stored backup evidence digest has changed")
+    if keyring_digest != verification.get("keyring_sha256"):
+        raise DistributionError("stored backup keyring digest has changed")
+    previous_release = receipt.get("release") if receipt else None
+    expected_release: Mapping[str, object] = (
+        previous_release
+        if isinstance(previous_release, Mapping)
+        else {
+            "channel": spec.release.channel,
+            "version": spec.release.version,
+            "manifest_sha256": spec.release.manifest_sha256,
+            "composition_sha256": spec.release.composition_sha256,
+            "api_image": spec.release.api_image,
+            "web_image": spec.release.web_image,
+        }
+    )
+    summary = verify_backup_evidence(
+        evidence,
+        keyring,
+        installation_id=spec.installation_id,
+        profile=spec.profile,
+        release=expected_release,
+    )
+    expected_summary = {
+        "signature_key_id": verification.get("signature_key_id"),
+        "evidence_id": verification.get("evidence_id"),
+        "recovery_point_id": verification.get("recovery_point_id"),
+        "restore_drill_id": verification.get("restore_drill_id"),
+        "captured_at": verification.get("captured_at"),
+        "expires_at": verification.get("expires_at"),
+        "restore_started_at": verification.get("restore_started_at"),
+        "restore_completed_at": verification.get("restore_completed_at"),
+        "measured_rpo_seconds": verification.get("measured_rpo_seconds"),
+        "measured_rto_seconds": verification.get("measured_rto_seconds"),
+        "component_count": verification.get("component_count"),
+    }
+    for field, expected in expected_summary.items():
+        if summary.get(field) != expected:
+            raise DistributionError(
+                f"backup verification receipt does not match {field!r}"
+            )
+    if expected_release.get("manifest_sha256") != verification.get(
+        "release_manifest_sha256"
+    ):
+        raise DistributionError("backup verification receipt has another release")
+    return {
+        **summary,
+        "evidence_sha256": evidence_digest,
+        "keyring_sha256": keyring_digest,
+    }
+
+
+def _backup_evidence_checks(
+    spec: InstallationSpec,
+    paths: BundlePaths,
+    *,
+    receipt: Mapping[str, object],
+) -> tuple[Check, ...]:
+    required = release_change_requires_backup(spec, receipt)
+    available = all(
+        path.is_file()
+        for path in (
+            paths.backup_evidence,
+            paths.backup_keyring,
+            paths.backup_verification,
+        )
+    )
+    if not available:
+        return (
+            Check(
+                "backup.migration_gate",
+                "error"
+                if required
+                else "warning"
+                if spec.profile == "self-hosted"
+                else "ok",
+                (
+                    "A release-changing migration has no verified coordinated backup evidence."
+                    if required
+                    else "No current coordinated backup evidence is adopted."
+                ),
+                (
+                    "Run verify-backup --adopt after an isolated restore drill."
+                    if spec.profile == "self-hosted"
+                    else ""
+                ),
+            ),
+        )
+    try:
+        summary = verify_stored_backup_evidence(
+            spec,
+            paths,
+            receipt=receipt,
+        )
+    except (DistributionError, OSError) as exc:
+        return (
+            Check(
+                "backup.migration_gate",
+                "error" if required else "warning",
+                f"Coordinated backup evidence is invalid: {exc}",
+                "Adopt fresh signed evidence for the currently applied release.",
+            ),
+        )
+    return (
+        Check(
+            "backup.migration_gate",
+            "ok",
+            (
+                "Release migration is backed by recovery point "
+                f"{summary['recovery_point_id']} and restore drill "
+                f"{summary['restore_drill_id']}."
+            ),
+        ),
+    )
+
+
 def _ingress_configuration_checks(
     spec: InstallationSpec,
     paths: BundlePaths,

tools/deployment/govoplan_deploy/recovery.py

@@ -29,6 +29,9 @@ _BUNDLE_FILES = (
     "existing-proxy.json",
     "distribution-manifest.json",
     "distribution-keyring.json",
+    "backup-evidence.json",
+    "backup-keyring.json",
+    "backup-verification.json",
     "receipt.json",
 )
 

tools/deployment/sign-backup-evidence.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Sign and validate provider-produced GovOPlaN backup evidence."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+from pathlib import Path
+import re
+import stat
+from typing import Any
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+from govoplan_deploy.backup_evidence import (
+    MAX_BACKUP_EVIDENCE_BYTES,
+    load_backup_keyring,
+    verify_backup_evidence,
+)
+from govoplan_deploy.bundle import atomic_write
+from govoplan_deploy.distribution import (
+    canonical_json,
+    canonical_signed_payload,
+    load_bounded_json,
+)
+
+
+KEY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Sign a provider-produced backup/restore evidence document and "
+            "validate it against an independently managed public keyring."
+        )
+    )
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--trusted-keyring", type=Path, required=True)
+    parser.add_argument(
+        "--signing-key",
+        action="append",
+        required=True,
+        metavar="KEY_ID=PRIVATE_PEM",
+        help="Ed25519 signer; may be repeated during key rotation.",
+    )
+    parser.add_argument(
+        "--replace-signatures",
+        action="store_true",
+        help="Replace existing signatures instead of rejecting the input.",
+    )
+    args = parser.parse_args()
+
+    source = args.input.expanduser().resolve()
+    payload = load_bounded_json(source, maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES)
+    existing = payload.get("signatures")
+    if existing not in (None, []) and not args.replace_signatures:
+        raise SystemExit("input already contains signatures; use --replace-signatures")
+
+    signers = [_load_signer(value) for value in args.signing_key]
+    if len({key_id for key_id, _ in signers}) != len(signers):
+        raise SystemExit("duplicate signing key id")
+    payload["signatures"] = []
+    signed = canonical_signed_payload(payload)
+    payload["signatures"] = [
+        {
+            "key_id": key_id,
+            "algorithm": "ed25519",
+            "value": base64.b64encode(private_key.sign(signed)).decode("ascii"),
+        }
+        for key_id, private_key in signers
+    ]
+
+    keyring = load_backup_keyring(args.trusted_keyring.expanduser().resolve())
+    release = payload.get("release")
+    if not isinstance(release, dict):
+        raise SystemExit("input release must be an object")
+    verify_backup_evidence(
+        payload,
+        keyring,
+        installation_id=str(payload.get("installation_id") or ""),
+        profile=str(
+            _object(payload.get("deployment_subject"), "deployment_subject").get(
+                "profile"
+            )
+            or ""
+        ),
+        release=release,
+    )
+    encoded = canonical_json(payload)
+    output = args.output.expanduser().resolve()
+    atomic_write(output, encoded, mode=0o600)
+    print(f"Wrote {output}")
+    print(f"SHA256 {hashlib.sha256(encoded).hexdigest()}")
+    return 0
+
+
+def _load_signer(value: str) -> tuple[str, Ed25519PrivateKey]:
+    key_id, separator, raw_path = value.partition("=")
+    if not separator or KEY_ID.fullmatch(key_id) is None or not raw_path:
+        raise SystemExit("--signing-key must use KEY_ID=/path/to/private.pem")
+    path = Path(raw_path).expanduser().resolve()
+    mode = stat.S_IMODE(path.stat().st_mode)
+    if mode & 0o077:
+        raise SystemExit(
+            f"private signing key must not be group/world accessible: {path}"
+        )
+    private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
+    if not isinstance(private_key, Ed25519PrivateKey):
+        raise SystemExit(f"signing key is not Ed25519: {path}")
+    return key_id, private_key
+
+
+def _object(value: object, label: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise SystemExit(f"input {label} must be an object")
+    return value
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/inventory/endpoint-surface-declarations.json

@@ -1581,6 +1581,134 @@
       "path": "/workflow/definitions/{}/triggers",
       "rationale": "This capability-first module intentionally exposes a headless API for other modules and integrations.",
       "repository": "govoplan-workflow-engine"
+    },
+    {
+      "category": "ui_reachable",
+      "method": "POST",
+      "path": "/addresses/contact-merges/{}/split",
+      "rationale": "The Addresses WebUI constructs the selected merge id and recovery action dynamically.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "ui_reachable",
+      "method": "POST",
+      "path": "/addresses/contact-merges/{}/undo",
+      "rationale": "The Addresses WebUI constructs the selected merge id and recovery action dynamically.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "intentionally_headless",
+      "method": "POST",
+      "path": "/addresses/contact-point-snapshots",
+      "rationale": "Campaign and other modules consume the governed contact-point snapshot capability without a direct Addresses screen.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "intentionally_headless",
+      "method": "GET",
+      "path": "/addresses/contact-point-snapshots/{}",
+      "rationale": "Campaign and other modules consume the governed contact-point snapshot capability without a direct Addresses screen.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "intentionally_headless",
+      "method": "POST",
+      "path": "/addresses/contact-point-sources/preview",
+      "rationale": "This capability endpoint previews a module-supplied contact source and is consumed by integrations rather than a direct screen.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "intentionally_headless",
+      "method": "POST",
+      "path": "/addresses/contact-points/resolve",
+      "rationale": "This governed recipient-resolution endpoint is consumed by Campaign and other modules.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "intentionally_headless",
+      "method": "GET",
+      "path": "/addresses/contacts/{}/redirect",
+      "rationale": "Stored references and module capabilities resolve merged-contact redirects without a direct user action.",
+      "repository": "govoplan-addresses"
+    },
+    {
+      "category": "missing_ui",
+      "method": "GET",
+      "path": "/addresses/imports/{}",
+      "rationale": "The import flow can create, apply, and roll back a run, but the WebUI does not yet resume a saved run by id after navigation or reload.",
+      "repository": "govoplan-addresses",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/22"
+    },
+    {
+      "category": "ui_reachable",
+      "method": "GET",
+      "path": "/campaigns/{}/versions/{}/print-output/download",
+      "rationale": "The Campaign WebUI validates and follows the build artifact's server-provided download path.",
+      "repository": "govoplan-campaign"
+    },
+    {
+      "category": "missing_ui",
+      "method": "GET",
+      "path": "/relationships",
+      "rationale": "IDM documents typed relationships but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "POST",
+      "path": "/relationships",
+      "rationale": "IDM documents typed relationships but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "PATCH",
+      "path": "/relationships/{}",
+      "rationale": "IDM documents typed relationships but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "POST",
+      "path": "/relationships/{}/revoke",
+      "rationale": "IDM documents typed relationships but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "GET",
+      "path": "/typed-groups",
+      "rationale": "IDM documents typed groups but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "POST",
+      "path": "/typed-groups",
+      "rationale": "IDM documents typed groups but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "PATCH",
+      "path": "/typed-groups/{}",
+      "rationale": "IDM documents typed groups but its current administration page exposes function assignments only.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
+    },
+    {
+      "category": "missing_ui",
+      "method": "GET",
+      "path": "/typed-groups/{}/memberships",
+      "rationale": "IDM lacks the typed-group membership explanation surface for this existing API.",
+      "repository": "govoplan-idm",
+      "tracking_issue": "https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/11"
     }
   ],
   "schema_version": 1

tools/release/publish-runtime-release.py

@@ -9,6 +9,7 @@ import json
 import mimetypes
 import os
 from pathlib import Path
+import re
 import secrets
 import sys
 from typing import Any
@@ -18,6 +19,7 @@ from urllib.request import Request, urlopen
 
 
 MAX_ASSET_BYTES = 256 * 1024 * 1024
+COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$")
 
 
 class PublishError(RuntimeError):
@@ -30,6 +32,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--owner", default="GovOPlaN")
     parser.add_argument("--repo", default="govoplan")
     parser.add_argument("--tag", required=True)
+    parser.add_argument("--target-commit", required=True)
     parser.add_argument("--title", required=True)
     parser.add_argument("--body", default="Signed GovOPlaN runtime distribution.")
     parser.add_argument("--asset", type=Path, action="append", default=[], required=True)
@@ -48,25 +51,49 @@ class GiteaReleasePublisher:
         self.repo = repo
         self.token = token
 
-    def release(self, *, tag: str, title: str, body: str) -> dict[str, Any]:
+    def release(
+        self,
+        *,
+        tag: str,
+        target_commit: str,
+        title: str,
+        body: str,
+    ) -> dict[str, Any]:
+        if COMMIT_SHA.fullmatch(target_commit) is None:
+            raise PublishError("release target must be an exact lowercase commit SHA")
+        resolved_target = self._resolve_commit(target_commit)
+        if resolved_target != target_commit:
+            raise PublishError("release target did not resolve to the requested commit")
+        existing_tag = self._resolve_commit(tag, allow_missing=True)
+        if existing_tag is not None and existing_tag != target_commit:
+            raise PublishError(
+                f"release tag {tag!r} already points to another commit"
+            )
+
         path = self._repo_path(f"/releases/tags/{quote(tag, safe='')}")
         try:
-            return self._json("GET", path)
+            release = self._json("GET", path)
         except HTTPError as exc:
             if exc.code != 404:
                 raise
-        return self._json(
-            "POST",
-            self._repo_path("/releases"),
-            payload={
-                "tag_name": tag,
-                "name": title,
-                "body": body,
-                "draft": False,
-                "prerelease": False,
-            },
-            expected=201,
-        )
+            release = self._json(
+                "POST",
+                self._repo_path("/releases"),
+                payload={
+                    "tag_name": tag,
+                    "target_commitish": target_commit,
+                    "name": title,
+                    "body": body,
+                    "draft": False,
+                    "prerelease": False,
+                },
+                expected=201,
+            )
+        if self._resolve_commit(tag) != target_commit:
+            raise PublishError(
+                f"release tag {tag!r} does not resolve to the requested commit"
+            )
+        return release
 
     def upload_assets(self, release: dict[str, Any], assets: tuple[Path, ...]) -> None:
         release_id = release.get("id")
@@ -165,6 +192,21 @@ class GiteaReleasePublisher:
     def _headers(self) -> dict[str, str]:
         return {"Authorization": f"token {self.token}", "Accept": "application/json"}
 
+    def _resolve_commit(self, ref: str, *, allow_missing: bool = False) -> str | None:
+        path = self._repo_path(f"/git/commits/{quote(ref, safe='')}")
+        try:
+            commit = self._json("GET", path)
+        except HTTPError as exc:
+            if allow_missing and exc.code == 404:
+                return None
+            raise
+        if not isinstance(commit, dict):
+            raise PublishError(f"Gitea returned an invalid commit for {ref!r}")
+        sha = commit.get("sha")
+        if not isinstance(sha, str) or COMMIT_SHA.fullmatch(sha) is None:
+            raise PublishError(f"Gitea returned an invalid commit SHA for {ref!r}")
+        return sha
+
     def _repo_path(self, suffix: str) -> str:
         return (
             f"{self.base_url}/api/v1/repos/{quote(self.owner, safe='')}/"
@@ -189,7 +231,12 @@ def main() -> int:
             repo=args.repo,
             token=os.environ.get(args.token_env, ""),
         )
-        release = publisher.release(tag=args.tag, title=args.title, body=args.body)
+        release = publisher.release(
+            tag=args.tag,
+            target_commit=args.target_commit,
+            title=args.title,
+            body=args.body,
+        )
         publisher.upload_assets(release, tuple(args.asset))
     except (HTTPError, OSError, PublishError, ValueError) as exc:
         print(f"error: {exc}", file=sys.stderr)

tools/release/resolve-oci-platforms.py

@@ -21,6 +21,12 @@ def resolve_platforms(
 ) -> dict[str, object]:
     if not repository or "@" in repository or any(value.isspace() for value in repository):
         raise ValueError("repository must be an unpinned OCI repository name")
+    last_slash = repository.rfind("/")
+    last_colon = repository.rfind(":")
+    if last_colon > last_slash:
+        repository = repository[:last_colon]
+    if not repository:
+        raise ValueError("repository must be an unpinned OCI repository name")
     if DIGEST.fullmatch(index_digest) is None:
         raise ValueError("index digest must be sha256:<hex>")
     if not isinstance(payload, dict) or not isinstance(payload.get("manifests"), list):

tools/release/runtime/Dockerfile.api

@@ -12,6 +12,7 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONPATH=/opt/govoplan/runtime \
     PATH=/opt/govoplan/runtime/bin:${PATH} \
+    GOVOPLAN_CORE_SOURCE_ROOT=/opt/govoplan/runtime/govoplan_core_runtime \
     HOME=/var/lib/govoplan
 
 COPY wheelhouse/ /opt/govoplan/wheels/

tools/release/runtime/nginx.conf

@@ -13,7 +13,10 @@ http {
     sendfile on;
     server_tokens off;
     client_body_temp_path /tmp/client_temp;
+    fastcgi_temp_path /tmp/fastcgi_temp;
     proxy_temp_path /tmp/proxy_temp;
+    scgi_temp_path /tmp/scgi_temp;
+    uwsgi_temp_path /tmp/uwsgi_temp;
 
     map $http_x_forwarded_proto $govoplan_forwarded_proto {
         default $scheme;

tools/repo/sync-python-environment.py

@@ -149,6 +149,7 @@ def main() -> int:
             requirements=requirements,
             python=python,
             local_requirements=local_requirements,
+            repair_requirements=environment.stale_requirements,
             force=args.force,
         )
 
@@ -250,6 +251,7 @@ def build_install_plan(
     python: str,
     local_requirements: tuple[RequirementEntry, ...],
     force: bool,
+    repair_requirements: tuple[RequirementEntry, ...] = (),
 ) -> InstallPlan:
     full_command = (python, "-m", "pip", "install", "-r", str(requirements))
     if force:
@@ -273,7 +275,12 @@ def build_install_plan(
     removed_paths = set(previous_inputs) - set(current_inputs)
     stale_requirements = requirements_key in changed_paths or requirements_key in removed_paths
 
-    installs: list[tuple[str, ...]] = []
+    # Metadata changes and installation drift can happen together. Keep both
+    # sets in one resolver transaction so a selective metadata sync also
+    # repairs local distributions omitted from the current environment.
+    installs: list[tuple[str, ...]] = [
+        requirement.install_args for requirement in repair_requirements
+    ]
     warnings: list[str] = []
 
     if stale_requirements:
@@ -322,7 +329,7 @@ def build_install_plan(
         )
         return InstallPlan(
             "Selective Python environment sync",
-            f"installing {len(deduped_installs)} stale local requirement(s) in one resolver transaction.",
+            f"installing {len(deduped_installs)} stale or missing local requirement(s) in one resolver transaction.",
             (command,),
             tuple(warnings),
         )