feat: implement institutional governance and recovery architecture

tools/deployment/govoplan_deploy/cli.py

@@ -47,7 +47,17 @@ from .model import (
     load_spec,
     parse_spec,
 )
+from .kubernetes import (
+    kubernetes_secret_contract,
+    render_kubernetes,
+    write_secret_creation_hint,
+)
 from .planning import DeploymentPlan, build_plan
+from .recovery import (
+    DeploymentOperationJournal,
+    list_operations,
+    recover_operation,
+)
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -115,6 +125,43 @@ def build_parser() -> argparse.ArgumentParser:
     )
     _directory_argument(status)
     status.add_argument("--json", action="store_true", help="Print JSON.")
+
+    kubernetes = subparsers.add_parser(
+        "render-kubernetes",
+        help="Export the stateless multi-host runtime for Kubernetes.",
+    )
+    _directory_argument(kubernetes)
+    kubernetes.add_argument("--namespace", default="govoplan")
+    kubernetes.add_argument("--secret-name", default="govoplan-runtime")
+    kubernetes.add_argument("--tls-secret-name", default="govoplan-tls")
+    kubernetes.add_argument("--ingress-class-name")
+    kubernetes.add_argument(
+        "--output",
+        type=Path,
+        help="Output JSON path; defaults to <directory>/kubernetes.json.",
+    )
+
+    operations = subparsers.add_parser(
+        "operations",
+        help="List durable deployment and recovery operations.",
+    )
+    _directory_argument(operations)
+    operations.add_argument("--json", action="store_true", help="Print JSON.")
+
+    recover = subparsers.add_parser(
+        "recover",
+        help="Restore a pre-migration applied bundle or enter forward recovery.",
+    )
+    _directory_argument(recover)
+    recover.add_argument("--operation-id")
+    recover.add_argument(
+        "--apply",
+        action="store_true",
+        help="Apply the restored or forward-recovery desired state immediately.",
+    )
+    recover.add_argument("--allow-unverified-images", action="store_true")
+    recover.add_argument("--skip-pull", action="store_true")
+    recover.add_argument("--health-timeout-seconds", type=float, default=120.0)
     return parser
 
 
@@ -233,6 +280,12 @@ def main(argv: Sequence[str] | None = None) -> int:
             return _apply(args)
         if args.command == "status":
             return _status(args)
+        if args.command == "render-kubernetes":
+            return _render_kubernetes(args)
+        if args.command == "operations":
+            return _operations(args)
+        if args.command == "recover":
+            return _recover(args)
     except (SpecError, ValueError, OSError, subprocess.SubprocessError) as exc:
         print(f"error: {exc}", file=sys.stderr)
         return 1
@@ -393,6 +446,10 @@ def _apply(args: argparse.Namespace) -> int:
         docker = shutil.which("docker")
         if docker is None:
             raise ValueError("Docker CLI is required for apply")
+        journal = DeploymentOperationJournal.begin(
+            paths,
+            plan=plan.to_dict(),
+        )
         compose = [
             docker,
             "compose",
@@ -403,62 +460,73 @@ def _apply(args: argparse.Namespace) -> int:
             "--file",
             str(paths.compose),
         ]
-        if not args.skip_pull:
-            _run([*compose, "pull"], cwd=paths.root)
-        dependencies = [
-            name
-            for name in ("postgres", "redis", "garage", "test-mail")
-            if name in service_names(spec)
-        ]
-        if dependencies:
-            _run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
-        _run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
-        if _receipt_uses_direct_web_port(paths.receipt):
-            _run([*compose, "stop", "web"], cwd=paths.root)
-        runtime_services = [
-            name
-            for name in ("api", "web", "load-balancer", "worker", "scheduler")
-            if name in service_names(spec)
-        ]
-        _run(
-            [*compose, "up", "--detach", "--remove-orphans", *runtime_services],
-            cwd=paths.root,
-        )
-        _wait_for_health(
-            f"{spec.public_url}/health",
-            timeout_seconds=args.health_timeout_seconds,
-        )
-        receipt = {
-            "schema_version": 1,
-            "installation_id": spec.installation_id,
-            "applied_at": _now(),
-            "spec_sha256": digest_json(spec.to_dict()),
-            "compose_sha256": digest_json(render_compose(spec)),
-            "environment_fingerprint": environment_fingerprint(secrets),
-            "release": {
-                "channel": spec.release.channel,
-                "version": spec.release.version,
-                "manifest_sha256": spec.release.manifest_sha256,
-                "api_image": spec.release.api_image,
-                "web_image": spec.release.web_image,
-            },
-            "services": list(service_names(spec)),
-            "replicas": {
-                "api": spec.replicas.api,
-                "web": spec.replicas.web,
-                "worker": spec.replicas.worker,
-            },
-            "listen": {
-                "address": spec.listen.address,
-                "port": spec.listen.port,
-            },
-            "management": {
-                "mode": "govoplan-deploy",
-                "agent": "cli",
-                "web_updates": False,
-            },
-        }
-        atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
+        try:
+            if not args.skip_pull:
+                _run([*compose, "pull"], cwd=paths.root)
+                journal.record("images-pulled", "succeeded")
+            dependencies = [
+                name
+                for name in ("postgres", "redis", "garage", "test-mail")
+                if name in service_names(spec)
+            ]
+            if dependencies:
+                _run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
+            journal.record(
+                "state-services-ready",
+                "succeeded",
+                {"services": dependencies},
+            )
+            mutable_runtime_services = [
+                name
+                for name in ("api", "worker", "scheduler")
+                if name in service_names(spec)
+            ]
+            if mutable_runtime_services:
+                _run(
+                    [
+                        *compose,
+                        "stop",
+                        "--timeout",
+                        "120",
+                        *mutable_runtime_services,
+                    ],
+                    cwd=paths.root,
+                )
+            journal.record(
+                "runtime-quiesced",
+                "succeeded",
+                {"services": mutable_runtime_services, "timeout_seconds": 120},
+            )
+            journal.migration_started()
+            _run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
+            journal.migration_completed()
+            if _receipt_uses_direct_web_port(paths.receipt):
+                _run([*compose, "stop", "web"], cwd=paths.root)
+            runtime_services = [
+                name
+                for name in ("api", "web", "load-balancer", "worker", "scheduler")
+                if name in service_names(spec)
+            ]
+            _run(
+                [*compose, "up", "--detach", "--remove-orphans", *runtime_services],
+                cwd=paths.root,
+            )
+            journal.record(
+                "runtime-reconciled",
+                "succeeded",
+                {"services": runtime_services},
+            )
+            _wait_for_health(
+                f"{spec.public_url}/health/ready",
+                timeout_seconds=args.health_timeout_seconds,
+            )
+            journal.record("health-verified", "succeeded", {"url": spec.public_url})
+            receipt = _deployment_receipt(spec, secrets)
+            atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
+            journal.succeeded(paths, receipt=receipt)
+        except BaseException as exc:
+            journal.failed(exc)
+            raise
         print(f"GovOPlaN is ready at {spec.public_url}")
         return 0
 
@@ -515,6 +583,104 @@ def _status(args: argparse.Namespace) -> int:
     return 0
 
 
+def _render_kubernetes(args: argparse.Namespace) -> int:
+    paths = bundle_paths(args.directory)
+    spec = load_spec(paths.spec)
+    environment = reconcile_runtime_environment(spec, read_env(paths.env))
+    manifest = render_kubernetes(
+        spec,
+        environment,
+        namespace=args.namespace,
+        secret_name=args.secret_name,
+        tls_secret_name=args.tls_secret_name,
+        ingress_class_name=args.ingress_class_name,
+    )
+    output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
+    atomic_write(output, canonical_json(manifest), mode=0o600)
+    print(f"Wrote stateless Kubernetes runtime manifest to {output}")
+    print(
+        "Required Secret keys: " + ", ".join(kubernetes_secret_contract())
+    )
+    print(
+        write_secret_creation_hint(
+            paths.env,
+            namespace=args.namespace,
+            secret_name=args.secret_name,
+        )
+    )
+    return 0
+
+
+def _operations(args: argparse.Namespace) -> int:
+    paths = bundle_paths(args.directory)
+    payload = list_operations(paths)
+    if args.json:
+        print(json.dumps(payload, indent=2, sort_keys=True))
+    elif not payload:
+        print("No deployment operations recorded.")
+    else:
+        for operation in payload:
+            print(
+                f"{operation['operation_id']}  {operation['status']}  "
+                f"{operation['recovery_mode']}  stages={operation['stage_count']}"
+            )
+            if operation.get("failure_summary"):
+                print(f"  {operation['failure_summary']}")
+    return 0
+
+
+def _recover(args: argparse.Namespace) -> int:
+    paths = bundle_paths(args.directory)
+    ensure_private_directory(paths.root)
+    with _deployment_lock(paths.lock):
+        result = recover_operation(paths, operation_id=args.operation_id)
+    print(f"Recovery operation {result.operation_id}: {result.action}")
+    print(result.detail)
+    if args.apply:
+        if not result.apply_allowed:
+            raise ValueError("this recovery state cannot be applied automatically")
+        return _apply(args)
+    if result.apply_allowed:
+        print(f"Reconcile with: govoplan-deploy apply --directory {paths.root}")
+    return 0
+
+
+def _deployment_receipt(
+    spec: InstallationSpec,
+    secrets: Mapping[str, str],
+) -> dict[str, object]:
+    return {
+        "schema_version": 1,
+        "installation_id": spec.installation_id,
+        "applied_at": _now(),
+        "spec_sha256": digest_json(spec.to_dict()),
+        "compose_sha256": digest_json(render_compose(spec)),
+        "environment_fingerprint": environment_fingerprint(secrets),
+        "release": {
+            "channel": spec.release.channel,
+            "version": spec.release.version,
+            "manifest_sha256": spec.release.manifest_sha256,
+            "api_image": spec.release.api_image,
+            "web_image": spec.release.web_image,
+        },
+        "services": list(service_names(spec)),
+        "replicas": {
+            "api": spec.replicas.api,
+            "web": spec.replicas.web,
+            "worker": spec.replicas.worker,
+        },
+        "listen": {
+            "address": spec.listen.address,
+            "port": spec.listen.port,
+        },
+        "management": {
+            "mode": "govoplan-deploy",
+            "agent": "cli",
+            "web_updates": False,
+        },
+    }
+
+
 def _updated_spec(
     current: InstallationSpec, args: argparse.Namespace
 ) -> InstallationSpec: