GovOPlaN/govoplan
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement institutional governance and recovery architecture
Dependency Audit / dependency-audit (push) Failing after 10s
Details
Deployment Installer / deployment-installer (push) Successful in 6s
Details
Security Audit / security-audit (push) Failing after 9s
Details

Browse Source
This commit is contained in:
zemion
2026-08-01 17:46:53 +02:00
parent 3c658fa32d
commit d78b13f9d3
45 changed files with 4388 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/deployment/govoplan_deploy/bundle.py
+56 -4
View File
@@ -28,6 +28,12 @@ LOCK_FILENAME = ".deployment.lock"
RUNTIME_ENV_KEYS = (
"APP_ENV",
"GOVOPLAN_INSTALL_PROFILE",
"GOVOPLAN_INSTALLATION_ID",
"GOVOPLAN_STATE_PROFILE",
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS",
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS",
"GOVOPLAN_EXPECTED_API_REPLICAS",
"GOVOPLAN_EXPECTED_WORKER_REPLICAS",
"MASTER_KEY_B64",
"DATABASE_URL",
"GOVOPLAN_DATABASE_URL_PGTOOLS",
@@ -55,6 +61,7 @@ RUNTIME_ENV_KEYS = (
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
)
@@ -169,6 +176,18 @@ def reconcile_runtime_environment(
{
"APP_ENV": "production" if spec.profile == "self-hosted" else "staging",
"GOVOPLAN_INSTALL_PROFILE": spec.profile,
"GOVOPLAN_INSTALLATION_ID": spec.installation_id,
# Generated Compose replicas share one Docker host. A deliberately
# Redis-free, single-API evaluation remains process-local; the
# Kubernetes exporter overrides this to ``shared`` for independent
# hosts.
"GOVOPLAN_STATE_PROFILE": (
"local" if redis.mode == "disabled" else "host-shared"
),
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "15",
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
"ENABLED_MODULES": ",".join(spec.enabled_modules),
"CELERY_ENABLED": "true" if redis.mode != "disabled" else "false",
"CELERY_QUEUES": (
@@ -198,6 +217,7 @@ def reconcile_runtime_environment(
if storage.mode == "local":
values["FILE_STORAGE_BACKEND"] = "local"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
values["FILE_STORAGE_S3_ENDPOINT_TRUSTED"] = "false"
values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files"
elif storage.mode == "garage":
access_key = values.setdefault(
@@ -221,11 +241,13 @@ def reconcile_runtime_environment(
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": secret_key,
"FILE_STORAGE_S3_BUCKET": bucket,
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "true",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "false",
}
)
else:
values["FILE_STORAGE_BACKEND"] = "s3"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
values["FILE_STORAGE_S3_ENDPOINT_TRUSTED"] = "true"
required = (
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_REGION",
@@ -243,12 +265,13 @@ def reconcile_runtime_environment(
schemes={"http", "https"},
label="S3 endpoint URL",
)
if spec.profile == "self-hosted" and endpoint.scheme != "https":
raise ValueError("self-hosted S3 endpoint URL must use HTTPS")
if endpoint.scheme != "https":
raise ValueError("external S3 endpoint URL must use HTTPS")
return dict(sorted(values.items()))
def render_compose(spec: InstallationSpec) -> dict[str, object]:
health_host = urlsplit(spec.public_url).hostname or "localhost"
runtime_env = {
"environment": _environment_references(RUNTIME_ENV_KEYS),
}
@@ -378,6 +401,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
services["migrate"] = {
**runtime_env,
"environment": {
**runtime_env["environment"],
"GOVOPLAN_RUNTIME_ROLE": "migration",
},
"image": spec.release.api_image,
"command": ["python", "-m", "govoplan_core.commands.init_db"],
"restart": "no",
@@ -387,6 +414,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
services["api"] = {
**common_runtime,
"environment": {
**runtime_env["environment"],
"GOVOPLAN_RUNTIME_ROLE": "api",
},
"image": spec.release.api_image,
"scale": spec.replicas.api,
"command": [
@@ -407,7 +438,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
"-c",
(
"import urllib.request;"
"urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)"
"request=urllib.request.Request("
"'http://127.0.0.1:8000/health/ready',"
f"headers={{'Host': {health_host!r}}});"
"urllib.request.urlopen(request,timeout=3)"
),
],
"interval": "10s",
@@ -454,6 +488,10 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
if spec.components.redis.mode != "disabled":
services["worker"] = {
**common_runtime,
"environment": {
**runtime_env["environment"],
"GOVOPLAN_RUNTIME_ROLE": "worker",
},
"image": spec.release.api_image,
"scale": spec.replicas.worker,
"command": [
@@ -474,8 +512,20 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
services["scheduler"] = {
**common_runtime,
"environment": {
**runtime_env["environment"],
"GOVOPLAN_RUNTIME_ROLE": "scheduler",
},
"image": spec.release.api_image,
"command": [
"python",
"-m",
"govoplan_core.commands.fenced_run",
"--resource",
"scheduler:celery-beat",
"--wait-seconds",
"120",
"--",
"python",
"-m",
"celery",
@@ -531,6 +581,7 @@ api_bind_addr = "[::]:3903"
def render_load_balancer_config(spec: InstallationSpec) -> str:
health_host = urlsplit(spec.public_url).hostname or "localhost"
return f"""global
log stdout format raw local0
maxconn 4096
@@ -572,7 +623,8 @@ frontend internal_api
backend api_replicas
balance leastconn
option httpchk GET /health
option httpchk GET /health/ready
http-check send hdr Host {health_host}
http-check expect status 200
server-template api- {spec.replicas.api} api:8000 check resolvers docker init-addr libc,none
"""
tools/deployment/govoplan_deploy/cli.py
+222 -56
View File
@@ -47,7 +47,17 @@ from .model import (
load_spec,
parse_spec,
)
from .kubernetes import (
kubernetes_secret_contract,
render_kubernetes,
write_secret_creation_hint,
)
from .planning import DeploymentPlan, build_plan
from .recovery import (
DeploymentOperationJournal,
list_operations,
recover_operation,
)
def build_parser() -> argparse.ArgumentParser:
@@ -115,6 +125,43 @@ def build_parser() -> argparse.ArgumentParser:
)
_directory_argument(status)
status.add_argument("--json", action="store_true", help="Print JSON.")
kubernetes = subparsers.add_parser(
"render-kubernetes",
help="Export the stateless multi-host runtime for Kubernetes.",
)
_directory_argument(kubernetes)
kubernetes.add_argument("--namespace", default="govoplan")
kubernetes.add_argument("--secret-name", default="govoplan-runtime")
kubernetes.add_argument("--tls-secret-name", default="govoplan-tls")
kubernetes.add_argument("--ingress-class-name")
kubernetes.add_argument(
"--output",
type=Path,
help="Output JSON path; defaults to <directory>/kubernetes.json.",
)
operations = subparsers.add_parser(
"operations",
help="List durable deployment and recovery operations.",
)
_directory_argument(operations)
operations.add_argument("--json", action="store_true", help="Print JSON.")
recover = subparsers.add_parser(
"recover",
help="Restore a pre-migration applied bundle or enter forward recovery.",
)
_directory_argument(recover)
recover.add_argument("--operation-id")
recover.add_argument(
"--apply",
action="store_true",
help="Apply the restored or forward-recovery desired state immediately.",
)
recover.add_argument("--allow-unverified-images", action="store_true")
recover.add_argument("--skip-pull", action="store_true")
recover.add_argument("--health-timeout-seconds", type=float, default=120.0)
return parser
@@ -233,6 +280,12 @@ def main(argv: Sequence[str] | None = None) -> int:
return _apply(args)
if args.command == "status":
return _status(args)
if args.command == "render-kubernetes":
return _render_kubernetes(args)
if args.command == "operations":
return _operations(args)
if args.command == "recover":
return _recover(args)
except (SpecError, ValueError, OSError, subprocess.SubprocessError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
@@ -393,6 +446,10 @@ def _apply(args: argparse.Namespace) -> int:
docker = shutil.which("docker")
if docker is None:
raise ValueError("Docker CLI is required for apply")
journal = DeploymentOperationJournal.begin(
paths,
plan=plan.to_dict(),
)
compose = [
docker,
"compose",
@@ -403,62 +460,73 @@ def _apply(args: argparse.Namespace) -> int:
"--file",
str(paths.compose),
]
if not args.skip_pull:
_run([*compose, "pull"], cwd=paths.root)
dependencies = [
name
for name in ("postgres", "redis", "garage", "test-mail")
if name in service_names(spec)
]
if dependencies:
_run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
if _receipt_uses_direct_web_port(paths.receipt):
_run([*compose, "stop", "web"], cwd=paths.root)
runtime_services = [
name
for name in ("api", "web", "load-balancer", "worker", "scheduler")
if name in service_names(spec)
]
_run(
[*compose, "up", "--detach", "--remove-orphans", *runtime_services],
cwd=paths.root,
)
_wait_for_health(
f"{spec.public_url}/health",
timeout_seconds=args.health_timeout_seconds,
)
receipt = {
"schema_version": 1,
"installation_id": spec.installation_id,
"applied_at": _now(),
"spec_sha256": digest_json(spec.to_dict()),
"compose_sha256": digest_json(render_compose(spec)),
"environment_fingerprint": environment_fingerprint(secrets),
"release": {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
},
"services": list(service_names(spec)),
"replicas": {
"api": spec.replicas.api,
"web": spec.replicas.web,
"worker": spec.replicas.worker,
},
"listen": {
"address": spec.listen.address,
"port": spec.listen.port,
},
"management": {
"mode": "govoplan-deploy",
"agent": "cli",
"web_updates": False,
},
}
atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
try:
if not args.skip_pull:
_run([*compose, "pull"], cwd=paths.root)
journal.record("images-pulled", "succeeded")
dependencies = [
name
for name in ("postgres", "redis", "garage", "test-mail")
if name in service_names(spec)
]
if dependencies:
_run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
journal.record(
"state-services-ready",
"succeeded",
{"services": dependencies},
)
mutable_runtime_services = [
name
for name in ("api", "worker", "scheduler")
if name in service_names(spec)
]
if mutable_runtime_services:
_run(
[
*compose,
"stop",
"--timeout",
"120",
*mutable_runtime_services,
],
cwd=paths.root,
)
journal.record(
"runtime-quiesced",
"succeeded",
{"services": mutable_runtime_services, "timeout_seconds": 120},
)
journal.migration_started()
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
journal.migration_completed()
if _receipt_uses_direct_web_port(paths.receipt):
_run([*compose, "stop", "web"], cwd=paths.root)
runtime_services = [
name
for name in ("api", "web", "load-balancer", "worker", "scheduler")
if name in service_names(spec)
]
_run(
[*compose, "up", "--detach", "--remove-orphans", *runtime_services],
cwd=paths.root,
)
journal.record(
"runtime-reconciled",
"succeeded",
{"services": runtime_services},
)
_wait_for_health(
f"{spec.public_url}/health/ready",
timeout_seconds=args.health_timeout_seconds,
)
journal.record("health-verified", "succeeded", {"url": spec.public_url})
receipt = _deployment_receipt(spec, secrets)
atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
journal.succeeded(paths, receipt=receipt)
except BaseException as exc:
journal.failed(exc)
raise
print(f"GovOPlaN is ready at {spec.public_url}")
return 0
@@ -515,6 +583,104 @@ def _status(args: argparse.Namespace) -> int:
return 0
def _render_kubernetes(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
spec = load_spec(paths.spec)
environment = reconcile_runtime_environment(spec, read_env(paths.env))
manifest = render_kubernetes(
spec,
environment,
namespace=args.namespace,
secret_name=args.secret_name,
tls_secret_name=args.tls_secret_name,
ingress_class_name=args.ingress_class_name,
)
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
atomic_write(output, canonical_json(manifest), mode=0o600)
print(f"Wrote stateless Kubernetes runtime manifest to {output}")
print(
"Required Secret keys: " + ", ".join(kubernetes_secret_contract())
)
print(
write_secret_creation_hint(
paths.env,
namespace=args.namespace,
secret_name=args.secret_name,
)
)
return 0
def _operations(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
payload = list_operations(paths)
if args.json:
print(json.dumps(payload, indent=2, sort_keys=True))
elif not payload:
print("No deployment operations recorded.")
else:
for operation in payload:
print(
f"{operation['operation_id']} {operation['status']} "
f"{operation['recovery_mode']} stages={operation['stage_count']}"
)
if operation.get("failure_summary"):
print(f" {operation['failure_summary']}")
return 0
def _recover(args: argparse.Namespace) -> int:
paths = bundle_paths(args.directory)
ensure_private_directory(paths.root)
with _deployment_lock(paths.lock):
result = recover_operation(paths, operation_id=args.operation_id)
print(f"Recovery operation {result.operation_id}: {result.action}")
print(result.detail)
if args.apply:
if not result.apply_allowed:
raise ValueError("this recovery state cannot be applied automatically")
return _apply(args)
if result.apply_allowed:
print(f"Reconcile with: govoplan-deploy apply --directory {paths.root}")
return 0
def _deployment_receipt(
spec: InstallationSpec,
secrets: Mapping[str, str],
) -> dict[str, object]:
return {
"schema_version": 1,
"installation_id": spec.installation_id,
"applied_at": _now(),
"spec_sha256": digest_json(spec.to_dict()),
"compose_sha256": digest_json(render_compose(spec)),
"environment_fingerprint": environment_fingerprint(secrets),
"release": {
"channel": spec.release.channel,
"version": spec.release.version,
"manifest_sha256": spec.release.manifest_sha256,
"api_image": spec.release.api_image,
"web_image": spec.release.web_image,
},
"services": list(service_names(spec)),
"replicas": {
"api": spec.replicas.api,
"web": spec.replicas.web,
"worker": spec.replicas.worker,
},
"listen": {
"address": spec.listen.address,
"port": spec.listen.port,
},
"management": {
"mode": "govoplan-deploy",
"agent": "cli",
"web_updates": False,
},
}
def _updated_spec(
current: InstallationSpec, args: argparse.Namespace
) -> InstallationSpec:
tools/deployment/govoplan_deploy/kubernetes.py
+666
View File
@@ -0,0 +1,666 @@
"""Deterministic Kubernetes export for the stateless multi-host profile."""
from __future__ import annotations
from hashlib import sha256
from pathlib import Path
import re
from typing import Any, Mapping
from urllib.parse import urlsplit
from .model import InstallationSpec, image_is_digest_pinned
_DNS_LABEL = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$")
_SECRET_KEYS = (
"MASTER_KEY_B64",
"DATABASE_URL",
"GOVOPLAN_DATABASE_URL_PGTOOLS",
"REDIS_URL",
"FILE_STORAGE_S3_ACCESS_KEY_ID",
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
)
_CONFIG_KEYS = (
"APP_ENV",
"GOVOPLAN_INSTALL_PROFILE",
"ENABLED_MODULES",
"CELERY_ENABLED",
"CELERY_QUEUES",
"CORS_ORIGINS",
"GOVOPLAN_TRUSTED_HOSTS",
"FORWARDED_ALLOW_IPS",
"AUTH_COOKIE_SECURE",
"AUTH_COOKIE_SAMESITE",
"GOVOPLAN_HTTP_HSTS_SECONDS",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
"GOVOPLAN_MIGRATION_TRACK",
"DEV_AUTO_MIGRATE_ENABLED",
"DEV_BOOTSTRAP_ENABLED",
"FILE_STORAGE_BACKEND",
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_REGION",
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
)
def render_kubernetes(
spec: InstallationSpec,
environment: Mapping[str, str],
*,
namespace: str = "govoplan",
secret_name: str = "govoplan-runtime",
tls_secret_name: str = "govoplan-tls",
ingress_class_name: str | None = None,
) -> dict[str, Any]:
"""Render runtime roles only; shared state services stay externally managed."""
_validate_cluster_profile(spec, environment, namespace, secret_name)
name = _resource_name(spec.installation_id)
public_host = urlsplit(spec.public_url).hostname or "localhost"
labels = {"app.kubernetes.io/name": "govoplan", "app.kubernetes.io/instance": name}
config_name = f"{name}-runtime"
service_account = f"{name}-runtime"
config = {
key: str(environment[key])
for key in _CONFIG_KEYS
if environment.get(key) is not None
}
config.update(
{
"GOVOPLAN_INSTALLATION_ID": spec.installation_id,
"GOVOPLAN_STATE_PROFILE": "shared",
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "15",
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "60",
"GOVOPLAN_EXPECTED_API_REPLICAS": str(spec.replicas.api),
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": str(spec.replicas.worker),
"FILE_STORAGE_BACKEND": "s3",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false",
"FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true",
# Trust no cross-pod proxy headers by default. Operators can bind
# an exact ingress-controller CIDR through their ConfigMap overlay.
"FORWARDED_ALLOW_IPS": "127.0.0.1",
}
)
items: list[dict[str, Any]] = [
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {"name": namespace, "labels": labels},
},
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": service_account,
"namespace": namespace,
"labels": labels,
},
"automountServiceAccountToken": False,
},
{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {"name": config_name, "namespace": namespace, "labels": labels},
"data": dict(sorted(config.items())),
},
_deployment(
name=f"{name}-api",
namespace=namespace,
labels=labels,
role="api",
replicas=spec.replicas.api,
image=spec.release.api_image,
command=(
"python",
"-m",
"uvicorn",
"govoplan_core.server.app:app",
"--host",
"0.0.0.0",
"--port",
"8000",
"--proxy-headers",
),
config_name=config_name,
secret_name=secret_name,
service_account=service_account,
container_port=8000,
readiness_path="/health/ready",
liveness_path="/health",
probe_host=public_host,
),
_service(
name=f"{name}-api",
namespace=namespace,
labels=labels,
role="api",
port=8000,
),
_deployment(
name=f"{name}-web",
namespace=namespace,
labels=labels,
role="web",
replicas=spec.replicas.web,
image=spec.release.web_image,
command=(),
config_name=None,
secret_name=None,
service_account=service_account,
container_port=8080,
extra_environment={"GOVOPLAN_API_UPSTREAM": f"http://{name}-api:8000"},
),
_service(
name=f"{name}-web",
namespace=namespace,
labels=labels,
role="web",
port=8080,
),
_migration_job(
name=name,
release_key=_release_key(spec),
namespace=namespace,
labels=labels,
image=spec.release.api_image,
config_name=config_name,
secret_name=secret_name,
service_account=service_account,
),
]
if spec.replicas.worker:
items.append(
_deployment(
name=f"{name}-worker",
namespace=namespace,
labels=labels,
role="worker",
replicas=spec.replicas.worker,
image=spec.release.api_image,
command=(
"python",
"-m",
"celery",
"-A",
"govoplan_core.celery_app:celery",
"worker",
"--queues",
str(environment["CELERY_QUEUES"]),
"--loglevel",
"INFO",
),
config_name=config_name,
secret_name=secret_name,
service_account=service_account,
)
)
items.append(
_deployment(
name=f"{name}-scheduler",
namespace=namespace,
labels=labels,
role="scheduler",
replicas=1,
image=spec.release.api_image,
command=(
"python",
"-m",
"govoplan_core.commands.fenced_run",
"--resource",
"scheduler:celery-beat",
"--wait-seconds",
"120",
"--",
"python",
"-m",
"celery",
"-A",
"govoplan_core.celery_app:celery",
"beat",
"--loglevel",
"INFO",
),
config_name=config_name,
secret_name=secret_name,
service_account=service_account,
)
)
if spec.replicas.api > 1:
items.append(_pod_disruption_budget(f"{name}-api", namespace, labels, "api"))
if spec.replicas.web > 1:
items.append(_pod_disruption_budget(f"{name}-web", namespace, labels, "web"))
items.append(
_ingress(
spec,
name=name,
namespace=namespace,
labels=labels,
tls_secret_name=tls_secret_name,
ingress_class_name=ingress_class_name,
)
)
return {
"apiVersion": "v1",
"kind": "List",
"metadata": {
"annotations": {
"govoplan.add-ideas.de/profile": "stateless-shared-state",
"govoplan.add-ideas.de/secret-contract": ",".join(_SECRET_KEYS),
}
},
"items": items,
}
def kubernetes_secret_contract() -> tuple[str, ...]:
return _SECRET_KEYS
def _validate_cluster_profile(
spec: InstallationSpec,
environment: Mapping[str, str],
namespace: str,
secret_name: str,
) -> None:
if not _DNS_LABEL.fullmatch(namespace) or not _DNS_LABEL.fullmatch(secret_name):
raise ValueError("Kubernetes namespace and secret names must be DNS labels")
if spec.installation_id == "govoplan-local":
raise ValueError(
"Kubernetes export requires a non-default stable installation id"
)
if spec.components.postgres.mode != "external":
raise ValueError("Kubernetes export requires external shared PostgreSQL")
if spec.components.redis.mode != "external":
raise ValueError("Kubernetes export requires external shared Redis")
if spec.components.storage.mode != "s3":
raise ValueError("Kubernetes export requires external shared S3 storage")
unpinned_images = [
label
for label, image in (
("release.api_image", spec.release.api_image),
("release.web_image", spec.release.web_image),
)
if not image_is_digest_pinned(image)
]
if unpinned_images:
raise ValueError(
"Kubernetes export requires immutable digest-pinned release images: "
+ ", ".join(unpinned_images)
)
missing = [
key for key in (*_SECRET_KEYS, "CELERY_QUEUES") if not environment.get(key)
]
if missing:
raise ValueError(
"Kubernetes runtime contract is missing: " + ", ".join(missing)
)
def _deployment(
*,
name: str,
namespace: str,
labels: dict[str, str],
role: str,
replicas: int,
image: str,
command: tuple[str, ...],
config_name: str | None,
secret_name: str | None,
service_account: str,
container_port: int | None = None,
readiness_path: str | None = None,
liveness_path: str | None = None,
probe_host: str | None = None,
extra_environment: Mapping[str, str] | None = None,
) -> dict[str, Any]:
role_labels = {**labels, "app.kubernetes.io/component": role}
environment: list[dict[str, Any]] = [
{"name": "TMPDIR", "value": "/tmp"},
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": role},
{
"name": "GOVOPLAN_NODE_ID",
"valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}},
},
]
environment.extend(
{"name": key, "value": value}
for key, value in sorted((extra_environment or {}).items())
)
if secret_name:
environment.extend(_secret_environment(secret_name))
container: dict[str, Any] = {
"name": role,
"image": image,
"imagePullPolicy": "IfNotPresent",
"env": environment,
"securityContext": {
"allowPrivilegeEscalation": False,
"capabilities": {"drop": ["ALL"]},
"readOnlyRootFilesystem": True,
},
"resources": {
"requests": {"cpu": "100m", "memory": "256Mi"},
"limits": {"memory": "1Gi"},
},
}
if command:
container["command"] = list(command)
if container_port is not None:
container["ports"] = [{"name": "http", "containerPort": container_port}]
if readiness_path:
container["readinessProbe"] = _http_probe(
readiness_path,
container_port or 8000,
host=probe_host,
)
if liveness_path:
container["livenessProbe"] = _http_probe(
liveness_path,
container_port or 8000,
host=probe_host,
)
pod_spec: dict[str, Any] = {
"serviceAccountName": service_account,
"automountServiceAccountToken": False,
"securityContext": {
"runAsNonRoot": True,
"seccompProfile": {"type": "RuntimeDefault"},
},
"containers": [container],
"volumes": [{"name": "tmp", "emptyDir": {}}],
"topologySpreadConstraints": [
{
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway",
"labelSelector": {"matchLabels": role_labels},
}
],
}
container["volumeMounts"] = [{"name": "tmp", "mountPath": "/tmp"}]
if config_name:
pod_spec["containers"][0]["envFrom"] = [{"configMapRef": {"name": config_name}}]
if config_name and secret_name:
pod_spec["initContainers"] = [
{
"name": "wait-for-database",
"image": image,
"imagePullPolicy": "IfNotPresent",
"command": [
"python",
"-m",
"govoplan_core.commands.wait_for_database",
"--timeout-seconds",
"900",
],
"envFrom": [{"configMapRef": {"name": config_name}}],
"env": [
{"name": "TMPDIR", "value": "/tmp"},
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration-wait"},
*_secret_environment(secret_name),
],
"securityContext": {
"allowPrivilegeEscalation": False,
"capabilities": {"drop": ["ALL"]},
"readOnlyRootFilesystem": True,
},
"volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
}
]
return {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
"spec": {
"replicas": replicas,
"strategy": {
"type": "RollingUpdate",
"rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1},
},
"selector": {"matchLabels": role_labels},
"template": {
"metadata": {"labels": role_labels},
"spec": pod_spec,
},
},
}
def _migration_job(
*,
name: str,
release_key: str,
namespace: str,
labels: dict[str, str],
image: str,
config_name: str,
secret_name: str,
service_account: str,
) -> dict[str, Any]:
job_labels = {**labels, "app.kubernetes.io/component": "migration"}
job_name = _name_with_suffix(name, f"migrate-{release_key}")
return {
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": job_name,
"namespace": namespace,
"labels": job_labels,
"annotations": {
"govoplan.add-ideas.de/recovery-mode": "forward-recovery",
"govoplan.add-ideas.de/backup-required": "true",
"argocd.argoproj.io/sync-wave": "-1",
},
},
"spec": {
"backoffLimit": 1,
"ttlSecondsAfterFinished": 86400,
"template": {
"metadata": {"labels": job_labels},
"spec": {
"restartPolicy": "Never",
"serviceAccountName": service_account,
"automountServiceAccountToken": False,
"securityContext": {
"runAsNonRoot": True,
"seccompProfile": {"type": "RuntimeDefault"},
},
"containers": [
{
"name": "migration",
"image": image,
"command": [
"python",
"-m",
"govoplan_core.commands.init_db",
],
"envFrom": [{"configMapRef": {"name": config_name}}],
"env": [
{"name": "TMPDIR", "value": "/tmp"},
{"name": "GOVOPLAN_RUNTIME_ROLE", "value": "migration"},
{
"name": "GOVOPLAN_NODE_ID",
"valueFrom": {
"fieldRef": {"fieldPath": "metadata.name"}
},
},
*_secret_environment(secret_name),
],
"securityContext": {
"allowPrivilegeEscalation": False,
"capabilities": {"drop": ["ALL"]},
"readOnlyRootFilesystem": True,
},
"volumeMounts": [{"name": "tmp", "mountPath": "/tmp"}],
}
],
"volumes": [{"name": "tmp", "emptyDir": {}}],
},
},
},
}
def _secret_environment(secret_name: str) -> list[dict[str, Any]]:
return [
{
"name": key,
"valueFrom": {
"secretKeyRef": {"name": secret_name, "key": key, "optional": False}
},
}
for key in _SECRET_KEYS
]
def _service(
*,
name: str,
namespace: str,
labels: dict[str, str],
role: str,
port: int,
) -> dict[str, Any]:
role_labels = {**labels, "app.kubernetes.io/component": role}
return {
"apiVersion": "v1",
"kind": "Service",
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
"spec": {
"selector": role_labels,
"ports": [{"name": "http", "port": port, "targetPort": "http"}],
},
}
def _pod_disruption_budget(
name: str,
namespace: str,
labels: dict[str, str],
role: str,
) -> dict[str, Any]:
role_labels = {**labels, "app.kubernetes.io/component": role}
return {
"apiVersion": "policy/v1",
"kind": "PodDisruptionBudget",
"metadata": {"name": name, "namespace": namespace, "labels": role_labels},
"spec": {"minAvailable": 1, "selector": {"matchLabels": role_labels}},
}
def _ingress(
spec: InstallationSpec,
*,
name: str,
namespace: str,
labels: dict[str, str],
tls_secret_name: str,
ingress_class_name: str | None,
) -> dict[str, Any]:
public = urlsplit(spec.public_url)
ingress_spec: dict[str, Any] = {
"rules": [
{
"host": public.hostname,
"http": {
"paths": [
{
"path": "/",
"pathType": "Prefix",
"backend": {
"service": {
"name": f"{name}-web",
"port": {"name": "http"},
}
},
}
]
},
}
]
}
if public.scheme == "https":
if not tls_secret_name:
raise ValueError("HTTPS Kubernetes export requires a TLS secret name")
ingress_spec["tls"] = [
{"hosts": [public.hostname], "secretName": tls_secret_name}
]
if ingress_class_name:
ingress_spec["ingressClassName"] = ingress_class_name
return {
"apiVersion": "networking.k8s.io/v1",
"kind": "Ingress",
"metadata": {"name": name, "namespace": namespace, "labels": labels},
"spec": ingress_spec,
}
def _http_probe(
path: str,
port: int,
*,
host: str | None = None,
) -> dict[str, Any]:
request: dict[str, Any] = {"path": path, "port": port}
if host:
request["httpHeaders"] = [{"name": "Host", "value": host}]
return {
"httpGet": request,
"initialDelaySeconds": 10,
"periodSeconds": 10,
"timeoutSeconds": 3,
"failureThreshold": 6,
}
def _resource_name(installation_id: str) -> str:
value = installation_id[:48].rstrip("-")
if not _DNS_LABEL.fullmatch(value):
raise ValueError("Installation id cannot be represented as a Kubernetes name")
return value
def _release_key(spec: InstallationSpec) -> str:
manifest_hash = str(spec.release.manifest_sha256 or "").strip().lower()
if re.fullmatch(r"[0-9a-f]{8,}", manifest_hash):
return manifest_hash[:8]
payload = "\0".join(
(
spec.release.version,
spec.release.api_image,
spec.release.web_image,
)
).encode("utf-8")
return sha256(payload).hexdigest()[:8]
def _name_with_suffix(name: str, suffix: str) -> str:
available = 63 - len(suffix) - 1
prefix = name[:available].rstrip("-")
candidate = f"{prefix}-{suffix}"
if not _DNS_LABEL.fullmatch(candidate):
raise ValueError("Kubernetes resource name cannot include the release suffix")
return candidate
def write_secret_creation_hint(
environment_path: Path,
*,
namespace: str,
secret_name: str,
) -> str:
keys = " ".join(f"--from-env-file={environment_path}" for _ in (0,))
return (
f"kubectl -n {namespace} create secret generic {secret_name} {keys} "
"--dry-run=client -o yaml | kubectl apply -f -"
)
__all__ = [
"kubernetes_secret_contract",
"render_kubernetes",
"write_secret_creation_hint",
]
tools/deployment/govoplan_deploy/recovery.py
+497
View File
@@ -0,0 +1,497 @@
"""Durable deployment operation journal and bounded bundle recovery."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from hashlib import sha256
import json
from pathlib import Path
import re
import shutil
from typing import Any
from uuid import uuid4
from .bundle import BundlePaths, atomic_write, canonical_json
OPERATIONS_DIRECTORY = "operations"
APPLIED_STATE_DIRECTORY = "applied-state"
OPERATION_FILENAME = "operation.json"
_OPERATION_ID = re.compile(r"^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$")
_BUNDLE_FILES = (
"installation.json",
"secrets.env",
"compose.json",
"garage.toml",
"load-balancer.cfg",
"receipt.json",
)
@dataclass(frozen=True, slots=True)
class RecoveryResult:
operation_id: str
action: str
detail: str
apply_allowed: bool
class DeploymentOperationJournal:
def __init__(self, operation_path: Path, payload: dict[str, Any]) -> None:
self.path = operation_path
self.payload = payload
@classmethod
def begin(
cls,
paths: BundlePaths,
*,
plan: dict[str, Any],
) -> DeploymentOperationJournal:
operations_root = paths.root / OPERATIONS_DIRECTORY
_private_directory(operations_root)
operation_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid4().hex[:8]
operation_path = operations_root / operation_id
_private_directory(operation_path)
snapshot_source = paths.root / APPLIED_STATE_DIRECTORY
snapshot_available = snapshot_source.is_dir()
if snapshot_available:
shutil.copytree(
snapshot_source,
operation_path / "before",
copy_function=_private_copy,
)
payload: dict[str, Any] = {
"schema_version": 2,
"operation_id": operation_id,
"installation_id": plan.get("installation_id"),
"status": "running",
"recovery_mode": (
"configuration_rollback" if snapshot_available else "forward_recovery"
),
"previous_applied_snapshot": snapshot_available,
"migration_started": False,
"migration_completed": False,
"started_at": _now(),
"updated_at": _now(),
"completed_at": None,
"failure_summary": None,
"desired": plan,
"stages": [],
"evidence_head_sha256": None,
}
journal = cls(operation_path, payload)
journal.record(
"plan-recorded",
"succeeded",
{
"previous_applied_snapshot": snapshot_available,
"desired_sha256": sha256(canonical_json(plan)).hexdigest(),
},
)
return journal
@property
def operation_id(self) -> str:
return str(self.payload["operation_id"])
def record(
self,
stage: str,
status: str,
evidence: dict[str, Any] | None = None,
) -> None:
stages = list(self.payload.get("stages") or [])
item = {
"sequence": len(stages) + 1,
"stage": stage,
"status": status,
"at": _now(),
"evidence": dict(evidence or {}),
"previous_sha256": self.payload.get("evidence_head_sha256"),
}
item["checkpoint_sha256"] = sha256(canonical_json(item)).hexdigest()
stages.append(item)
self.payload["stages"] = stages
self.payload["evidence_head_sha256"] = item["checkpoint_sha256"]
self.payload["updated_at"] = item["at"]
self._write()
def migration_started(self) -> None:
self.payload["migration_started"] = True
self.payload["recovery_mode"] = "forward_recovery"
self.record(
"migration-started",
"running",
{
"rollback_boundary": (
"Database migration has started; automatic release/configuration rollback is disabled."
)
},
)
def migration_completed(self) -> None:
self.payload["migration_completed"] = True
self.record("migration-completed", "succeeded")
def succeeded(self, paths: BundlePaths, *, receipt: dict[str, Any]) -> None:
write_applied_state(paths)
self.payload["status"] = "succeeded"
self.payload["completed_at"] = _now()
self.record(
"deployment-verified",
"succeeded",
{
"receipt_spec_sha256": receipt.get("spec_sha256"),
"receipt_compose_sha256": receipt.get("compose_sha256"),
},
)
def failed(self, exc: BaseException) -> None:
self.payload["status"] = "failed"
self.payload["completed_at"] = _now()
self.payload["failure_summary"] = f"{type(exc).__name__}: {str(exc)[:1000]}"
self.record(
"deployment-failed",
"failed",
{
"exception_type": type(exc).__name__,
"recovery_mode": self.payload.get("recovery_mode"),
"failure_summary_sha256": sha256(
str(self.payload["failure_summary"]).encode("utf-8")
).hexdigest(),
},
)
def _write(self) -> None:
atomic_write(
self.path / OPERATION_FILENAME,
canonical_json(self.payload),
mode=0o600,
)
def write_applied_state(paths: BundlePaths) -> Path:
target = paths.root / APPLIED_STATE_DIRECTORY
temporary = paths.root / f".{APPLIED_STATE_DIRECTORY}.{uuid4().hex}.tmp"
previous = paths.root / f".{APPLIED_STATE_DIRECTORY}.previous"
_private_directory(temporary)
manifest: dict[str, Any] = {"schema_version": 1, "captured_at": _now(), "files": {}}
try:
for filename in _BUNDLE_FILES:
source = paths.root / filename
if not source.is_file():
manifest["files"][filename] = {"present": False}
continue
destination = temporary / filename
_private_copy(source, destination)
manifest["files"][filename] = {
"present": True,
"sha256": sha256(destination.read_bytes()).hexdigest(),
}
atomic_write(
temporary / "manifest.json",
canonical_json(manifest),
mode=0o600,
)
if previous.exists() and not target.exists():
previous.replace(target)
elif previous.exists():
shutil.rmtree(previous)
if target.exists():
target.replace(previous)
try:
temporary.replace(target)
except BaseException:
if previous.exists() and not target.exists():
previous.replace(target)
raise
if previous.exists():
shutil.rmtree(previous)
finally:
if temporary.exists():
shutil.rmtree(temporary)
return target
def list_operations(paths: BundlePaths) -> list[dict[str, Any]]:
root = paths.root / OPERATIONS_DIRECTORY
if not root.is_dir():
return []
operations: list[dict[str, Any]] = []
for path in sorted(root.iterdir(), reverse=True):
if not path.is_dir() or not _OPERATION_ID.fullmatch(path.name):
continue
payload = _read_operation(path)
operations.append(
{
"operation_id": payload.get("operation_id"),
"status": payload.get("status"),
"recovery_mode": payload.get("recovery_mode"),
"migration_started": bool(payload.get("migration_started")),
"started_at": payload.get("started_at"),
"completed_at": payload.get("completed_at"),
"failure_summary": payload.get("failure_summary"),
"stage_count": len(payload.get("stages") or []),
}
)
return operations
def recover_operation(
paths: BundlePaths,
*,
operation_id: str | None = None,
) -> RecoveryResult:
operation_path, payload = load_operation(paths, operation_id=operation_id)
effective_id = str(payload["operation_id"])
if bool(payload.get("migration_started")):
journal = DeploymentOperationJournal(operation_path, payload)
journal.payload["status"] = "forward_recovery_required"
journal.payload["recovery_mode"] = "forward_recovery"
journal.record(
"forward-recovery-required",
"blocked",
{
"reason": "migration_started",
"configuration_restored": False,
},
)
return RecoveryResult(
operation_id=effective_id,
action="forward_recovery",
detail=(
"Database migration started. The previous release/configuration was not restored; "
"repair the current release or restore a separately verified database backup."
),
apply_allowed=True,
)
before = operation_path / "before"
if not before.is_dir():
return RecoveryResult(
operation_id=effective_id,
action="manual_intervention",
detail="No independently captured previous applied state is available.",
apply_allowed=False,
)
manifest = json.loads((before / "manifest.json").read_text(encoding="utf-8"))
entries = _validate_snapshot(before, manifest)
for filename, evidence in entries:
target = paths.root / filename
if evidence.get("present"):
source = before / filename
_private_copy(source, target)
else:
target.unlink(missing_ok=True)
journal = DeploymentOperationJournal(operation_path, payload)
journal.payload["status"] = "configuration_restored"
journal.payload["recovery_mode"] = "configuration_rollback"
journal.record(
"configuration-restored",
"succeeded",
{"snapshot_manifest_verified": True},
)
return RecoveryResult(
operation_id=effective_id,
action="configuration_restored",
detail=(
"The previously applied bundle was restored and checksum-verified. "
"Run apply to reconcile containers to that state."
),
apply_allowed=True,
)
def load_operation(
paths: BundlePaths,
*,
operation_id: str | None = None,
) -> tuple[Path, dict[str, Any]]:
root = paths.root / OPERATIONS_DIRECTORY
if operation_id is None:
candidates = (
[
item
for item in sorted(root.iterdir(), reverse=True)
if item.is_dir() and _OPERATION_ID.fullmatch(item.name)
]
if root.is_dir()
else []
)
if not candidates:
raise ValueError("No deployment operations are recorded")
path = candidates[0]
else:
if not _OPERATION_ID.fullmatch(operation_id):
raise ValueError("Invalid deployment operation id")
path = root / operation_id
return path, _read_operation(path)
def _read_operation(path: Path) -> dict[str, Any]:
try:
payload = json.loads((path / OPERATION_FILENAME).read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError) as exc:
raise ValueError(f"Deployment operation is unreadable: {path.name}") from exc
if not isinstance(payload, dict) or payload.get("operation_id") != path.name:
raise ValueError(f"Deployment operation identity is invalid: {path.name}")
verify_operation_chain(payload)
return payload
def verify_operation_chain(payload: dict[str, Any]) -> None:
stages = payload.get("stages")
if not isinstance(stages, list) or not stages:
raise ValueError("Deployment operation has no recovery evidence chain")
previous: str | None = None
observed_migration_started = False
observed_migration_completed = False
for sequence, raw_stage in enumerate(stages, start=1):
if not isinstance(raw_stage, dict):
raise ValueError("Deployment recovery evidence contains an invalid stage")
stage = dict(raw_stage)
checkpoint = str(stage.pop("checkpoint_sha256", ""))
if (
stage.get("sequence") != sequence
or stage.get("previous_sha256") != previous
or sha256(canonical_json(stage)).hexdigest() != checkpoint
):
raise ValueError(
f"Deployment recovery evidence chain failed at stage {sequence}"
)
previous = checkpoint
observed_migration_started |= stage.get("stage") == "migration-started"
observed_migration_completed |= stage.get("stage") == "migration-completed"
if previous != payload.get("evidence_head_sha256"):
raise ValueError("Deployment recovery evidence head does not match its chain")
if bool(payload.get("migration_started")) != observed_migration_started:
raise ValueError(
"Deployment migration boundary does not match recovery evidence"
)
if bool(payload.get("migration_completed")) != observed_migration_completed:
raise ValueError(
"Deployment migration completion does not match recovery evidence"
)
if int(payload.get("schema_version") or 0) >= 2:
_verify_operation_state(payload, stages)
def _verify_operation_state(
payload: dict[str, Any],
stages: list[dict[str, Any]],
) -> None:
plan_evidence = stages[0].get("evidence")
if (
stages[0].get("stage") != "plan-recorded"
or not isinstance(plan_evidence, dict)
or plan_evidence.get("desired_sha256")
!= sha256(canonical_json(payload.get("desired"))).hexdigest()
or bool(plan_evidence.get("previous_applied_snapshot"))
!= bool(payload.get("previous_applied_snapshot"))
):
raise ValueError("Deployment plan does not match recovery evidence")
desired = payload.get("desired")
if not isinstance(desired, dict) or desired.get("installation_id") != payload.get(
"installation_id"
):
raise ValueError("Deployment installation identity does not match its plan")
status = str(payload.get("status") or "")
expected_last_stage = {
"failed": "deployment-failed",
"succeeded": "deployment-verified",
"configuration_restored": "configuration-restored",
"forward_recovery_required": "forward-recovery-required",
}.get(status)
if expected_last_stage and stages[-1].get("stage") != expected_last_stage:
raise ValueError("Deployment status does not match recovery evidence")
if status == "failed":
failure_summary = str(payload.get("failure_summary") or "")
evidence = stages[-1].get("evidence")
if (
not failure_summary
or not isinstance(evidence, dict)
or evidence.get("failure_summary_sha256")
!= sha256(failure_summary.encode("utf-8")).hexdigest()
):
raise ValueError(
"Deployment failure summary does not match recovery evidence"
)
if status in {"failed", "succeeded"} and not payload.get("completed_at"):
raise ValueError("Completed deployment operation has no completion timestamp")
migration_started = bool(payload.get("migration_started"))
if bool(payload.get("migration_completed")) and not migration_started:
raise ValueError("Completed migration has no recorded migration boundary")
expected_mode = (
"forward_recovery"
if migration_started or not bool(payload.get("previous_applied_snapshot"))
else "configuration_rollback"
)
if status == "configuration_restored":
expected_mode = "configuration_rollback"
if payload.get("recovery_mode") != expected_mode:
raise ValueError("Deployment recovery mode does not match recovery evidence")
def _validate_snapshot(
before: Path,
manifest: object,
) -> list[tuple[str, dict[str, Any]]]:
if not isinstance(manifest, dict) or manifest.get("schema_version") != 1:
raise ValueError("Applied-state snapshot manifest is invalid")
files = manifest.get("files")
if not isinstance(files, dict) or set(files) != set(_BUNDLE_FILES):
raise ValueError("Applied-state snapshot manifest is incomplete")
entries: list[tuple[str, dict[str, Any]]] = []
for filename in _BUNDLE_FILES:
evidence = files.get(filename)
if not isinstance(evidence, dict) or not isinstance(
evidence.get("present"),
bool,
):
raise ValueError(
f"Applied-state snapshot evidence is invalid for {filename}"
)
if evidence["present"]:
source = before / filename
expected = str(evidence.get("sha256") or "")
if (
source.is_symlink()
or not source.is_file()
or not re.fullmatch(r"[0-9a-f]{64}", expected)
or sha256(source.read_bytes()).hexdigest() != expected
):
raise ValueError(
f"Applied-state snapshot checksum failed for {filename}"
)
entries.append((filename, evidence))
return entries
def _private_directory(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True, mode=0o700)
path.chmod(0o700)
def _private_copy(source: Path, destination: Path) -> Path:
source = Path(source)
destination = Path(destination)
if source.is_symlink() or not source.is_file():
raise ValueError(f"Refusing to copy non-regular bundle file: {source.name}")
atomic_write(destination, source.read_bytes(), mode=0o600)
return destination
def _now() -> str:
return datetime.now(UTC).isoformat()
__all__ = [
"DeploymentOperationJournal",
"RecoveryResult",
"list_operations",
"load_operation",
"recover_operation",
"verify_operation_chain",
"write_applied_state",
]