diff --git a/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md b/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md index b06f0e8..69f37ce 100644 --- a/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md +++ b/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md @@ -44,6 +44,10 @@ least one check. The ledger verifies its hash chain before evidence is trusted. This is a platform contract, not an assertion that every existing module operation has adopted it. Module operations with external or multi-resource effects must be migrated to the ledger before claiming these guarantees. +The owning-module inventory and adoption state are maintained in +[Recovery Ledger Adoption](RECOVERY_LEDGER_ADOPTION.md); CI validates the +machine-readable inventory so newly identified boundaries cannot disappear from +the backlog silently. ## Deployment Journal diff --git a/docs/RECOVERY_LEDGER_ADOPTION.md b/docs/RECOVERY_LEDGER_ADOPTION.md new file mode 100644 index 0000000..ffa932c --- /dev/null +++ b/docs/RECOVERY_LEDGER_ADOPTION.md @@ -0,0 +1,56 @@ +# Recovery Ledger Adoption + +The Core recovery ledger is a platform primitive, not automatic protection for +module-owned effects. The canonical, machine-checked inventory is +[`recovery-operation-inventory.json`](recovery-operation-inventory.json). + +## Classification Rules + +- Use `atomic` only when every mutation commits in one database transaction and + no external effect occurs. +- Use `compensation` when every completed effect has a bounded, verifiable + inverse action. A best-effort delete is not proof of compensation. +- Use `snapshot_restore` only with fresh, signed backup evidence that covers all + affected state services at one recovery point. +- Use `forward_recovery` for provider acceptance, queue publication, cursor + advancement, and other effects that may be resumable but cannot safely be + undone. +- Use `irreversible` for approved purge or destruction where no automated + recovery is claimed. + +One feature may cross more than one boundary. Module installation is +compensatable before schema migration, forward-only after migration starts, and +snapshot-restorable for an approved destructive retirement. Mail submission is +forward recovery because losing the response after provider acceptance must not +cause an automatic resend. + +## Adoption Order + +1. Campaign build is the reference implementation for a database plus object + storage operation. Its operation reserves a build-specific object prefix, + persists request and precondition evidence before writes, records the final + object manifest, and verifies database/object state before success. +2. Campaign delivery and Mail provider effects adopt outcome-unknown semantics + without weakening their existing provider-specific idempotency records. +3. Files applies the same contract to uploads, purge, integrity reconciliation, + and writable connector synchronization. +4. Connectors, Dataflow, and Workflow Engine consume the contract at their + registry/capability boundaries so optional providers remain optional. +5. Core module lifecycle uses the ledger in addition to, not instead of, signed + deployment and backup evidence. + +Every fenced operation uses a process incarnation and distributed lease. A +stale process cannot append a checkpoint or report success. An expired operation +is claimed for recovery through an explicit takeover that preserves the prior +fence in the checkpoint chain; it is never resumed as a normal retry. + +## Operator Contract + +Ops lists non-terminal and manual-intervention operations. Operators must verify +the checkpoint chain before trusting evidence, distinguish `outcome_unknown` +from rejection, and use the owning module's documented reconciliation action. +No evidence payload may contain credentials or resolved secrets. + +The parent adoption issue remains open until all inventory rows are adopted and +the module matrix proves crash, retry, stale-fence, tamper, and optional-module +behavior for each consequential path. diff --git a/docs/recovery-operation-inventory.json b/docs/recovery-operation-inventory.json new file mode 100644 index 0000000..fffcd58 --- /dev/null +++ b/docs/recovery-operation-inventory.json @@ -0,0 +1,177 @@ +{ + "schema_version": 1, + "parent_issue": "https://git.add-ideas.de/GovOPlaN/govoplan/issues/36", + "operations": [ + { + "id": "campaign.build.publish-artifacts", + "repository": "govoplan-campaign", + "resources": ["postgresql", "object-storage", "templates-capability", "files-capability"], + "mode": "compensation", + "fenced": true, + "adoption": "reference-implementation", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92" + }, + { + "id": "campaign.delivery.external-channels", + "repository": "govoplan-campaign", + "resources": ["postgresql", "queue", "smtp", "imap", "postbox", "print-provider"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92" + }, + { + "id": "campaign.retention.generated-artifacts", + "repository": "govoplan-campaign", + "resources": ["postgresql", "object-storage"], + "mode": "compensation", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92" + }, + { + "id": "files.upload.finalize", + "repository": "govoplan-files", + "resources": ["postgresql", "object-storage", "filesystem-staging"], + "mode": "compensation", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41" + }, + { + "id": "files.retention.purge", + "repository": "govoplan-files", + "resources": ["postgresql", "object-storage", "encryption-key-custody"], + "mode": "irreversible", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41" + }, + { + "id": "files.integrity.reconcile", + "repository": "govoplan-files", + "resources": ["postgresql", "object-storage"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41" + }, + { + "id": "files.connector.write-sync", + "repository": "govoplan-files", + "resources": ["postgresql", "object-storage", "external-connector"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41" + }, + { + "id": "mail.outbox.smtp-submit", + "repository": "govoplan-mail", + "resources": ["postgresql", "queue", "smtp"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19" + }, + { + "id": "mail.mailbox.imap-mutate", + "repository": "govoplan-mail", + "resources": ["postgresql", "imap"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19" + }, + { + "id": "mail.mailbox.sync-cursor", + "repository": "govoplan-mail", + "resources": ["postgresql", "imap", "object-storage"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19" + }, + { + "id": "connectors.sync.read-snapshot", + "repository": "govoplan-connectors", + "resources": ["postgresql", "external-provider"], + "mode": "atomic", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15" + }, + { + "id": "connectors.sync.external-mutation", + "repository": "govoplan-connectors", + "resources": ["postgresql", "queue", "external-provider"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15" + }, + { + "id": "dataflow.run.database-only", + "repository": "govoplan-dataflow", + "resources": ["postgresql"], + "mode": "atomic", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19" + }, + { + "id": "dataflow.run.publish-output", + "repository": "govoplan-dataflow", + "resources": ["postgresql", "queue", "object-storage", "external-sink"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19" + }, + { + "id": "workflow-engine.instance.state-transition", + "repository": "govoplan-workflow-engine", + "resources": ["postgresql"], + "mode": "atomic", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1" + }, + { + "id": "workflow-engine.activity.external-effect", + "repository": "govoplan-workflow-engine", + "resources": ["postgresql", "queue", "module-capability", "external-provider"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1" + }, + { + "id": "core.module-lifecycle.pre-migration", + "repository": "govoplan-core", + "resources": ["postgresql", "package-environment", "webui-bundle", "filesystem"], + "mode": "compensation", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281" + }, + { + "id": "core.module-lifecycle.post-migration", + "repository": "govoplan-core", + "resources": ["postgresql", "package-environment", "webui-bundle", "runtime-nodes"], + "mode": "forward_recovery", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281" + }, + { + "id": "core.module-retirement.destroy-data", + "repository": "govoplan-core", + "resources": ["postgresql", "object-storage", "package-environment"], + "mode": "snapshot_restore", + "fenced": true, + "adoption": "planned", + "issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281" + } + ] +} diff --git a/tests/test_recovery_operation_inventory.py b/tests/test_recovery_operation_inventory.py new file mode 100644 index 0000000..5451a67 --- /dev/null +++ b/tests/test_recovery_operation_inventory.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +from pathlib import Path +from urllib.parse import urlparse + + +ROOT = Path(__file__).resolve().parents[1] +INVENTORY = ROOT / "docs" / "recovery-operation-inventory.json" +MODES = { + "atomic", + "compensation", + "snapshot_restore", + "forward_recovery", + "irreversible", +} +ADOPTION_STATES = {"planned", "reference-implementation", "adopted"} +REQUIRED_PREFIXES = { + "campaign.", + "files.", + "mail.", + "connectors.", + "dataflow.", + "workflow-engine.", + "core.module-lifecycle.", +} + + +def test_recovery_operation_inventory_is_complete_and_actionable() -> None: + payload = json.loads(INVENTORY.read_text(encoding="utf-8")) + assert payload["schema_version"] == 1 + operations = payload["operations"] + ids = [item["id"] for item in operations] + assert len(ids) == len(set(ids)) + assert all(any(item.startswith(prefix) for item in ids) for prefix in REQUIRED_PREFIXES) + + for item in operations: + assert item["mode"] in MODES + assert item["adoption"] in ADOPTION_STATES + assert item["repository"].startswith("govoplan-") + assert item["resources"] + assert item["fenced"] is True + issue = urlparse(item["issue"]) + assert issue.scheme == "https" + assert issue.netloc == "git.add-ideas.de" + assert issue.path.startswith(f"/GovOPlaN/{item['repository']}/issues/") + + +def test_non_atomic_operations_do_not_claim_plain_database_rollback() -> None: + operations = json.loads(INVENTORY.read_text(encoding="utf-8"))["operations"] + for item in operations: + if item["mode"] == "atomic": + assert item["resources"] == ["postgresql"] or item["id"] == ( + "connectors.sync.read-snapshot" + )