1898 lines
66 KiB
Python
1898 lines
66 KiB
Python
"""Command-line interface for GovOPlaN deployment reconciliation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from dataclasses import replace
|
|
from datetime import UTC, datetime
|
|
import fcntl
|
|
import getpass
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Iterator, Mapping, Sequence
|
|
from urllib.error import URLError
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .backup_evidence import (
|
|
DEFAULT_MAX_BACKUP_AGE_SECONDS,
|
|
MAX_BACKUP_EVIDENCE_BYTES,
|
|
MAX_BACKUP_KEYRING_BYTES,
|
|
verify_backup_evidence,
|
|
)
|
|
from .bundle import (
|
|
BACKUP_RUNTIME_ENV_KEYS,
|
|
BundlePaths,
|
|
atomic_write,
|
|
bundle_paths,
|
|
canonical_json,
|
|
digest_json,
|
|
environment_fingerprint,
|
|
ensure_private_directory,
|
|
initial_secrets,
|
|
read_env,
|
|
reconcile_runtime_environment,
|
|
render_caddy_config,
|
|
render_compose,
|
|
render_existing_proxy_contract,
|
|
render_garage_config,
|
|
render_load_balancer_config,
|
|
service_names,
|
|
write_env,
|
|
)
|
|
from .capabilities import (
|
|
InfrastructureDependencyInventory,
|
|
infrastructure_capability_document,
|
|
infrastructure_dependency_inventory_from_mapping,
|
|
)
|
|
from .cluster_evidence import collect_kubernetes_evidence
|
|
from .distribution import (
|
|
MAX_KEYRING_BYTES,
|
|
MAX_MANIFEST_BYTES,
|
|
MAX_OFFLINE_INDEX_BYTES,
|
|
DistributionError,
|
|
canonical_json as canonical_distribution_json,
|
|
decode_json_bytes,
|
|
fetch_bounded_https,
|
|
load_bounded_json,
|
|
read_bounded_bytes,
|
|
verify_offline_image_index,
|
|
verify_manifest,
|
|
verify_manifest_binding,
|
|
)
|
|
from .model import (
|
|
ComponentConfig,
|
|
DEFAULT_GARAGE_IMAGE,
|
|
DEFAULT_INGRESS_IMAGE,
|
|
DEFAULT_LOAD_BALANCER_IMAGE,
|
|
IngressConfig,
|
|
InstallationSpec,
|
|
ListenConfig,
|
|
ReplicaConfig,
|
|
SpecError,
|
|
default_spec,
|
|
load_spec,
|
|
parse_spec,
|
|
)
|
|
from .kubernetes import (
|
|
kubernetes_secret_contract,
|
|
render_kubernetes,
|
|
write_secret_creation_hint,
|
|
)
|
|
from .planning import (
|
|
MAX_DEPENDENCY_INVENTORY_BYTES,
|
|
DeploymentPlan,
|
|
build_plan,
|
|
release_change_requires_backup,
|
|
verify_stored_backup_evidence,
|
|
)
|
|
from .recovery import (
|
|
DeploymentOperationJournal,
|
|
list_operations,
|
|
recover_operation,
|
|
)
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="govoplan-deploy",
|
|
description=(
|
|
"Create, validate, and reconcile a declarative GovOPlaN Compose deployment."
|
|
),
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
init = subparsers.add_parser(
|
|
"init", help="Create a new private installation bundle."
|
|
)
|
|
_directory_argument(init)
|
|
_configuration_arguments(init)
|
|
init.add_argument(
|
|
"--non-interactive",
|
|
action="store_true",
|
|
help="Use flags/defaults without prompting.",
|
|
)
|
|
|
|
configure = subparsers.add_parser(
|
|
"configure",
|
|
help="Change installation choices while preserving generated secrets.",
|
|
)
|
|
_directory_argument(configure)
|
|
_configuration_arguments(configure, defaults=False)
|
|
|
|
render = subparsers.add_parser(
|
|
"render", help="Regenerate Compose and the deployment plan without applying."
|
|
)
|
|
_directory_argument(render)
|
|
render.add_argument("--json", action="store_true", help="Print the plan as JSON.")
|
|
|
|
doctor = subparsers.add_parser(
|
|
"doctor", help="Run specification, secret, host, and provenance checks."
|
|
)
|
|
_directory_argument(doctor)
|
|
doctor.add_argument("--json", action="store_true", help="Print the plan as JSON.")
|
|
|
|
apply_parser = subparsers.add_parser(
|
|
"apply", help="Apply an allowed plan with Docker Compose."
|
|
)
|
|
_directory_argument(apply_parser)
|
|
apply_parser.add_argument(
|
|
"--allow-unverified-images",
|
|
action="store_true",
|
|
help="Allow mutable/unverified images for an evaluation profile only.",
|
|
)
|
|
apply_parser.add_argument(
|
|
"--skip-pull",
|
|
action="store_true",
|
|
help="Do not pull images before reconciliation.",
|
|
)
|
|
apply_parser.add_argument(
|
|
"--health-timeout-seconds",
|
|
type=float,
|
|
default=120.0,
|
|
help="Maximum time to wait for the public health endpoint.",
|
|
)
|
|
apply_parser.add_argument(
|
|
"--ops-url",
|
|
help=(
|
|
"Dependency inventory URL; defaults to "
|
|
"<public-url>/api/v1/ops/infrastructure/dependencies."
|
|
),
|
|
)
|
|
apply_parser.add_argument(
|
|
"--api-key-env",
|
|
default="GOVOPLAN_OPS_API_KEY",
|
|
help=(
|
|
"Environment variable containing an API key authorized to read "
|
|
"Ops dependency inventory."
|
|
),
|
|
)
|
|
|
|
collect_inventory = subparsers.add_parser(
|
|
"collect-infrastructure-inventory",
|
|
help="Collect current module-owned capability dependencies from Ops.",
|
|
)
|
|
_directory_argument(collect_inventory)
|
|
collect_inventory.add_argument(
|
|
"--ops-url",
|
|
help=(
|
|
"Dependency inventory URL; defaults to "
|
|
"<public-url>/api/v1/ops/infrastructure/dependencies."
|
|
),
|
|
)
|
|
collect_inventory.add_argument(
|
|
"--api-key-env",
|
|
default="GOVOPLAN_OPS_API_KEY",
|
|
help="Environment variable containing an authorized Ops API key.",
|
|
)
|
|
|
|
status = subparsers.add_parser(
|
|
"status", help="Show desired state and current Compose process state."
|
|
)
|
|
_directory_argument(status)
|
|
status.add_argument("--json", action="store_true", help="Print JSON.")
|
|
|
|
verify_release = subparsers.add_parser(
|
|
"verify-release",
|
|
help="Verify and optionally adopt a signed runtime distribution.",
|
|
)
|
|
_directory_argument(verify_release)
|
|
manifest_source = verify_release.add_mutually_exclusive_group(required=True)
|
|
manifest_source.add_argument("--manifest", type=Path)
|
|
manifest_source.add_argument("--manifest-url")
|
|
verify_release.add_argument(
|
|
"--manifest-sha256",
|
|
required=True,
|
|
help="Independently obtained SHA-256 digest of the signed manifest.",
|
|
)
|
|
verify_release.add_argument("--trusted-keyring", type=Path, required=True)
|
|
verify_release.add_argument(
|
|
"--allow-private-release-host",
|
|
action="store_true",
|
|
help="Allow an explicitly selected private HTTPS release mirror.",
|
|
)
|
|
verify_release.add_argument(
|
|
"--adopt",
|
|
action="store_true",
|
|
help="Store the verified trust material and select its pinned images.",
|
|
)
|
|
|
|
offline_images = subparsers.add_parser(
|
|
"verify-offline-images",
|
|
help="Verify prefetched OCI archives against the adopted distribution.",
|
|
)
|
|
_directory_argument(offline_images)
|
|
offline_images.add_argument("--index", type=Path, required=True)
|
|
offline_images.add_argument(
|
|
"--load",
|
|
action="store_true",
|
|
help="Load verified archives into Docker using fixed image-load commands.",
|
|
)
|
|
|
|
verify_backup = subparsers.add_parser(
|
|
"verify-backup",
|
|
help="Verify and optionally adopt signed coordinated backup evidence.",
|
|
)
|
|
_directory_argument(verify_backup)
|
|
evidence_source = verify_backup.add_mutually_exclusive_group(required=True)
|
|
evidence_source.add_argument("--evidence", type=Path)
|
|
evidence_source.add_argument("--evidence-url")
|
|
verify_backup.add_argument("--evidence-sha256", required=True)
|
|
verify_backup.add_argument("--trusted-keyring", type=Path, required=True)
|
|
verify_backup.add_argument(
|
|
"--allow-private-evidence-host",
|
|
action="store_true",
|
|
)
|
|
verify_backup.add_argument("--adopt", action="store_true")
|
|
|
|
kubernetes = subparsers.add_parser(
|
|
"render-kubernetes",
|
|
help="Export the stateless multi-host runtime for Kubernetes.",
|
|
)
|
|
_directory_argument(kubernetes)
|
|
kubernetes.add_argument("--namespace", default="govoplan")
|
|
kubernetes.add_argument("--secret-name", default="govoplan-runtime")
|
|
kubernetes.add_argument("--tls-secret-name", default="govoplan-tls")
|
|
kubernetes.add_argument(
|
|
"--s3-ca-secret-name",
|
|
help=(
|
|
"Optional Secret containing ca.crt for the external S3 endpoint; "
|
|
"mounted read-only into backend runtime roles."
|
|
),
|
|
)
|
|
kubernetes.add_argument("--ingress-class-name")
|
|
kubernetes.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
help="Output JSON path; defaults to <directory>/kubernetes.json.",
|
|
)
|
|
|
|
verify_kubernetes = subparsers.add_parser(
|
|
"verify-kubernetes",
|
|
help="Collect sanitized readiness evidence from a live multi-host profile.",
|
|
)
|
|
_directory_argument(verify_kubernetes)
|
|
verify_kubernetes.add_argument("--namespace", default="govoplan")
|
|
verify_kubernetes.add_argument(
|
|
"--ops-url",
|
|
help="Ops status URL; defaults to <public-url>/api/v1/ops/status.",
|
|
)
|
|
verify_kubernetes.add_argument(
|
|
"--api-key-env",
|
|
default="GOVOPLAN_OPS_API_KEY",
|
|
help=(
|
|
"Environment variable containing an API key authorized to read "
|
|
"Ops status."
|
|
),
|
|
)
|
|
verify_kubernetes.add_argument(
|
|
"--exercise-api-pod-loss",
|
|
action="store_true",
|
|
help="Delete one ready API pod and prove health-aware replacement.",
|
|
)
|
|
verify_kubernetes.add_argument(
|
|
"--timeout-seconds",
|
|
type=float,
|
|
default=180.0,
|
|
)
|
|
verify_kubernetes.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
help="Private evidence output path.",
|
|
)
|
|
|
|
operations = subparsers.add_parser(
|
|
"operations",
|
|
help="List durable deployment and recovery operations.",
|
|
)
|
|
_directory_argument(operations)
|
|
operations.add_argument("--json", action="store_true", help="Print JSON.")
|
|
|
|
recover = subparsers.add_parser(
|
|
"recover",
|
|
help="Restore a pre-migration applied bundle or enter forward recovery.",
|
|
)
|
|
_directory_argument(recover)
|
|
recover.add_argument("--operation-id")
|
|
recover.add_argument(
|
|
"--apply",
|
|
action="store_true",
|
|
help="Apply the restored or forward-recovery desired state immediately.",
|
|
)
|
|
recover.add_argument("--allow-unverified-images", action="store_true")
|
|
recover.add_argument("--skip-pull", action="store_true")
|
|
recover.add_argument("--health-timeout-seconds", type=float, default=120.0)
|
|
return parser
|
|
|
|
|
|
def _directory_argument(parser: argparse.ArgumentParser) -> None:
|
|
parser.add_argument(
|
|
"--directory",
|
|
type=Path,
|
|
default=Path.home()
|
|
/ ".local"
|
|
/ "share"
|
|
/ "govoplan"
|
|
/ "installations"
|
|
/ "default",
|
|
help="Private installation state directory.",
|
|
)
|
|
|
|
|
|
def _configuration_arguments(
|
|
parser: argparse.ArgumentParser, *, defaults: bool = True
|
|
) -> None:
|
|
default = (lambda value: value) if defaults else (lambda _value: None)
|
|
parser.add_argument("--installation-id", default=default("govoplan-local"))
|
|
parser.add_argument(
|
|
"--profile",
|
|
choices=("evaluation", "self-hosted"),
|
|
default=default("evaluation"),
|
|
)
|
|
parser.add_argument("--public-url", default=default("http://127.0.0.1:8080"))
|
|
parser.add_argument("--listen-address", default=default("127.0.0.1"))
|
|
parser.add_argument("--listen-port", type=int, default=default(8080))
|
|
parser.add_argument(
|
|
"--postgres",
|
|
choices=("managed", "external"),
|
|
default=default("managed"),
|
|
)
|
|
parser.add_argument(
|
|
"--database-url", help="External PostgreSQL URL; stored privately."
|
|
)
|
|
parser.add_argument(
|
|
"--redis",
|
|
choices=("managed", "external", "disabled"),
|
|
default=default("managed"),
|
|
)
|
|
parser.add_argument("--redis-url", help="External Redis URL; stored privately.")
|
|
parser.add_argument(
|
|
"--mail",
|
|
choices=("disabled", "external-relay", "test-mail"),
|
|
default=default("disabled"),
|
|
)
|
|
parser.add_argument(
|
|
"--storage",
|
|
choices=("local", "garage", "s3"),
|
|
default=default("local"),
|
|
)
|
|
parser.add_argument(
|
|
"--garage-image",
|
|
default=default(DEFAULT_GARAGE_IMAGE),
|
|
help="Garage image used by managed object storage.",
|
|
)
|
|
parser.add_argument("--s3-endpoint-url", help="S3-compatible endpoint URL.")
|
|
parser.add_argument("--s3-region", help="S3 region.")
|
|
parser.add_argument(
|
|
"--s3-access-key-id", help="S3 access key id; stored privately."
|
|
)
|
|
parser.add_argument(
|
|
"--s3-secret-access-key",
|
|
help="S3 secret access key; stored privately.",
|
|
)
|
|
parser.add_argument("--s3-bucket", help="S3 bucket for managed files.")
|
|
parser.add_argument(
|
|
"--load-balancer-image",
|
|
default=default(DEFAULT_LOAD_BALANCER_IMAGE),
|
|
help="HAProxy image used by the managed local load balancer.",
|
|
)
|
|
parser.add_argument(
|
|
"--ingress",
|
|
choices=("local", "existing-proxy", "managed", "unconfigured"),
|
|
default=default(None),
|
|
help="Public route boundary; self-hosted requires existing-proxy or managed.",
|
|
)
|
|
parser.add_argument(
|
|
"--ingress-image",
|
|
default=default(DEFAULT_INGRESS_IMAGE),
|
|
help="Caddy image used by managed ingress.",
|
|
)
|
|
parser.add_argument(
|
|
"--trusted-proxy-cidr",
|
|
action="append",
|
|
default=None,
|
|
help="Exact source CIDR trusted to supply forwarded headers; repeatable.",
|
|
)
|
|
parser.add_argument("--acme-email", default=default(""))
|
|
parser.add_argument("--ingress-http-port", type=int, default=default(80))
|
|
parser.add_argument("--ingress-https-port", type=int, default=default(443))
|
|
parser.add_argument(
|
|
"--api-replicas",
|
|
type=int,
|
|
default=default(1),
|
|
help="API containers on this Compose host (1-64).",
|
|
)
|
|
parser.add_argument(
|
|
"--web-replicas",
|
|
type=int,
|
|
default=default(1),
|
|
help="WebUI containers on this Compose host (1-64).",
|
|
)
|
|
parser.add_argument(
|
|
"--worker-replicas",
|
|
type=int,
|
|
default=None,
|
|
help="Worker containers on this Compose host (1-128, or 0 without Redis).",
|
|
)
|
|
parser.add_argument(
|
|
"--module-set",
|
|
choices=("core", "base", "full"),
|
|
default=default("base"),
|
|
)
|
|
parser.add_argument("--api-image", default=default("govoplan-api:unpublished"))
|
|
parser.add_argument("--web-image", default=default("govoplan-web:unpublished"))
|
|
parser.add_argument("--release-version", default=default("unpublished"))
|
|
parser.add_argument("--release-channel", default=default("stable"))
|
|
parser.add_argument("--manifest-url", default=default(""))
|
|
parser.add_argument("--manifest-sha256", default=default(""))
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
try:
|
|
if args.command == "init":
|
|
return _init(args)
|
|
if args.command == "configure":
|
|
return _configure(args)
|
|
if args.command in {"render", "doctor"}:
|
|
return _render_or_doctor(args)
|
|
if args.command == "apply":
|
|
return _apply(args)
|
|
if args.command == "collect-infrastructure-inventory":
|
|
return _collect_infrastructure_inventory(args)
|
|
if args.command == "status":
|
|
return _status(args)
|
|
if args.command == "verify-release":
|
|
return _verify_release(args)
|
|
if args.command == "verify-offline-images":
|
|
return _verify_offline_images(args)
|
|
if args.command == "verify-backup":
|
|
return _verify_backup(args)
|
|
if args.command == "render-kubernetes":
|
|
return _render_kubernetes(args)
|
|
if args.command == "verify-kubernetes":
|
|
return _verify_kubernetes(args)
|
|
if args.command == "operations":
|
|
return _operations(args)
|
|
if args.command == "recover":
|
|
return _recover(args)
|
|
except (
|
|
DistributionError,
|
|
SpecError,
|
|
ValueError,
|
|
OSError,
|
|
subprocess.SubprocessError,
|
|
) as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 1
|
|
raise RuntimeError(f"unsupported command: {args.command}")
|
|
|
|
|
|
def _init(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
ensure_private_directory(paths.root)
|
|
if paths.spec.exists():
|
|
raise ValueError(
|
|
f"{paths.spec} already exists; use configure for an existing installation"
|
|
)
|
|
if not args.non_interactive and sys.stdin.isatty():
|
|
_prompt_configuration(args)
|
|
spec = default_spec(
|
|
installation_id=args.installation_id,
|
|
profile=args.profile,
|
|
public_url=args.public_url,
|
|
listen_address=args.listen_address,
|
|
listen_port=args.listen_port,
|
|
postgres_mode=args.postgres,
|
|
redis_mode=args.redis,
|
|
mail_mode=args.mail,
|
|
storage_mode=args.storage,
|
|
garage_image=args.garage_image,
|
|
load_balancer_image=args.load_balancer_image,
|
|
ingress_mode=args.ingress,
|
|
ingress_image=args.ingress_image,
|
|
trusted_proxy_cidrs=tuple(args.trusted_proxy_cidr or ()),
|
|
ingress_http_port=args.ingress_http_port,
|
|
ingress_https_port=args.ingress_https_port,
|
|
acme_email=args.acme_email,
|
|
api_replicas=args.api_replicas,
|
|
web_replicas=args.web_replicas,
|
|
worker_replicas=args.worker_replicas,
|
|
module_set=args.module_set,
|
|
api_image=args.api_image,
|
|
web_image=args.web_image,
|
|
manifest_url=args.manifest_url,
|
|
manifest_sha256=args.manifest_sha256,
|
|
version=args.release_version,
|
|
channel=args.release_channel,
|
|
)
|
|
supplied = _supplied_secret_values(args)
|
|
secrets = initial_secrets(spec, supplied=supplied)
|
|
_write_bundle(spec, paths, secrets)
|
|
plan = build_plan(spec, paths, include_host_checks=False)
|
|
_write_plan(paths.plan, plan)
|
|
print(f"Created GovOPlaN installation bundle at {paths.root}")
|
|
print(f"Edit choices with: govoplan-deploy configure --directory {paths.root}")
|
|
print(f"Check readiness with: govoplan-deploy doctor --directory {paths.root}")
|
|
if any(check.level == "error" for check in plan.checks):
|
|
print("The bundle is not apply-ready yet; run doctor for the blocking checks.")
|
|
return 0
|
|
|
|
|
|
def _configure(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
ensure_private_directory(paths.root)
|
|
current = load_spec(paths.spec)
|
|
if args.installation_id and args.installation_id != current.installation_id:
|
|
raise ValueError(
|
|
"installation_id is immutable; create a separate installation "
|
|
"instead of renaming a Compose project"
|
|
)
|
|
spec = _updated_spec(current, args)
|
|
if (
|
|
current.components.postgres.mode == "managed"
|
|
and spec.components.postgres.mode == "external"
|
|
and not args.database_url
|
|
):
|
|
raise ValueError(
|
|
"switching PostgreSQL from managed to external requires --database-url"
|
|
)
|
|
if (
|
|
current.components.redis.mode != "external"
|
|
and spec.components.redis.mode == "external"
|
|
and not args.redis_url
|
|
):
|
|
raise ValueError("switching Redis to external requires --redis-url")
|
|
if current.components.storage.mode != "s3" and spec.components.storage.mode == "s3":
|
|
missing_s3_options = [
|
|
flag
|
|
for flag, value in (
|
|
("--s3-endpoint-url", args.s3_endpoint_url),
|
|
("--s3-region", args.s3_region),
|
|
("--s3-access-key-id", args.s3_access_key_id),
|
|
("--s3-secret-access-key", args.s3_secret_access_key),
|
|
("--s3-bucket", args.s3_bucket),
|
|
)
|
|
if not value
|
|
]
|
|
if missing_s3_options:
|
|
raise ValueError(
|
|
"switching to external S3 requires: " + ", ".join(missing_s3_options)
|
|
)
|
|
secrets = read_env(paths.env)
|
|
secrets.update(_supplied_secret_values(args))
|
|
secrets = reconcile_runtime_environment(spec, secrets)
|
|
_write_bundle(spec, paths, secrets)
|
|
plan = build_plan(spec, paths, include_host_checks=False)
|
|
_write_plan(paths.plan, plan)
|
|
print(f"Updated desired state at {paths.root}")
|
|
_print_plan(plan)
|
|
return 0
|
|
|
|
|
|
def _render_or_doctor(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
secrets = reconcile_runtime_environment(spec, read_env(paths.env))
|
|
_write_bundle(spec, paths, secrets)
|
|
plan = build_plan(
|
|
spec,
|
|
paths,
|
|
include_host_checks=args.command == "doctor",
|
|
)
|
|
_write_plan(paths.plan, plan)
|
|
if args.json:
|
|
print(json.dumps(plan.to_dict(), indent=2, sort_keys=True))
|
|
else:
|
|
_print_plan(plan)
|
|
return 1 if plan.blocked else 0
|
|
|
|
|
|
def _apply(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
ensure_private_directory(paths.root)
|
|
with _deployment_lock(paths.lock):
|
|
spec = load_spec(paths.spec)
|
|
previous_receipt = _read_json_object(paths.receipt)
|
|
backup_required = release_change_requires_backup(spec, previous_receipt)
|
|
if args.allow_unverified_images and spec.profile != "evaluation":
|
|
raise ValueError(
|
|
"--allow-unverified-images is restricted to evaluation installations"
|
|
)
|
|
secrets = reconcile_runtime_environment(spec, read_env(paths.env))
|
|
secrets = _write_bundle(spec, paths, secrets)
|
|
preliminary_plan = build_plan(spec, paths, include_host_checks=False)
|
|
api_key_env = str(
|
|
getattr(args, "api_key_env", "GOVOPLAN_OPS_API_KEY")
|
|
).strip()
|
|
api_key = os.environ.get(api_key_env, "").strip()
|
|
if preliminary_plan.capability_impacts and api_key:
|
|
try:
|
|
_collect_dependency_inventory(
|
|
spec,
|
|
paths,
|
|
ops_url=getattr(args, "ops_url", None),
|
|
api_key=api_key,
|
|
)
|
|
print("Refreshed infrastructure dependency inventory from Ops.")
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
print(
|
|
f"warning: could not refresh dependency inventory: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
plan = build_plan(spec, paths, include_host_checks=True)
|
|
_write_plan(paths.plan, plan)
|
|
effective_errors = [
|
|
check
|
|
for check in plan.checks
|
|
if check.level == "error"
|
|
and not (
|
|
args.allow_unverified_images
|
|
and check.id.startswith(
|
|
(
|
|
"release.api_image.",
|
|
"release.web_image.",
|
|
"components.postgres.image.",
|
|
"components.redis.image.",
|
|
"components.mail.image.",
|
|
"components.storage.image.",
|
|
"components.load_balancer.image.",
|
|
"ingress.image.",
|
|
"release.manifest",
|
|
"modules.image_composition",
|
|
)
|
|
)
|
|
)
|
|
]
|
|
if effective_errors:
|
|
_print_plan(plan)
|
|
raise ValueError("deployment plan is blocked; resolve doctor errors first")
|
|
if plan.capability_impacts:
|
|
_print_plan(plan)
|
|
docker = shutil.which("docker")
|
|
if docker is None:
|
|
raise ValueError("Docker CLI is required for apply")
|
|
journal = DeploymentOperationJournal.begin(
|
|
paths,
|
|
plan=plan.to_dict(),
|
|
)
|
|
try:
|
|
if backup_required:
|
|
backup_summary = verify_stored_backup_evidence(
|
|
spec,
|
|
paths,
|
|
receipt=previous_receipt,
|
|
)
|
|
journal.record(
|
|
"backup-evidence-verified",
|
|
"succeeded",
|
|
dict(backup_summary),
|
|
)
|
|
else:
|
|
journal.record(
|
|
"backup-evidence-not-required",
|
|
"succeeded",
|
|
{"release_change": False},
|
|
)
|
|
except BaseException as exc:
|
|
journal.record(
|
|
"backup-evidence-rejected",
|
|
"blocked",
|
|
{
|
|
"phase": "preflight",
|
|
"exception_type": type(exc).__name__,
|
|
"migration_started": False,
|
|
},
|
|
)
|
|
journal.failed(exc)
|
|
raise
|
|
compose = [
|
|
docker,
|
|
"compose",
|
|
"--env-file",
|
|
str(paths.env),
|
|
"--project-name",
|
|
spec.installation_id,
|
|
"--file",
|
|
str(paths.compose),
|
|
]
|
|
try:
|
|
if not args.skip_pull:
|
|
_run([*compose, "pull"], cwd=paths.root)
|
|
journal.record("images-pulled", "succeeded")
|
|
dependencies = [
|
|
name
|
|
for name in ("postgres", "redis", "garage", "test-mail")
|
|
if name in service_names(spec)
|
|
]
|
|
if dependencies:
|
|
_run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
|
|
journal.record(
|
|
"state-services-ready",
|
|
"succeeded",
|
|
{"services": dependencies},
|
|
)
|
|
mutable_runtime_services = [
|
|
name
|
|
for name in ("api", "worker", "scheduler")
|
|
if name in service_names(spec)
|
|
]
|
|
if mutable_runtime_services:
|
|
_run(
|
|
[
|
|
*compose,
|
|
"stop",
|
|
"--timeout",
|
|
"120",
|
|
*mutable_runtime_services,
|
|
],
|
|
cwd=paths.root,
|
|
)
|
|
journal.record(
|
|
"runtime-quiesced",
|
|
"succeeded",
|
|
{"services": mutable_runtime_services, "timeout_seconds": 120},
|
|
)
|
|
if backup_required:
|
|
try:
|
|
backup_summary = verify_stored_backup_evidence(
|
|
spec,
|
|
paths,
|
|
receipt=previous_receipt,
|
|
)
|
|
except BaseException as exc:
|
|
journal.record(
|
|
"backup-evidence-rejected",
|
|
"blocked",
|
|
{
|
|
"phase": "migration-boundary",
|
|
"exception_type": type(exc).__name__,
|
|
"migration_started": False,
|
|
},
|
|
)
|
|
raise
|
|
journal.record(
|
|
"backup-evidence-reverified",
|
|
"succeeded",
|
|
dict(backup_summary),
|
|
)
|
|
journal.migration_started()
|
|
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
|
|
journal.migration_completed()
|
|
if _receipt_uses_direct_web_port(paths.receipt):
|
|
_run([*compose, "stop", "web"], cwd=paths.root)
|
|
runtime_services = [
|
|
name
|
|
for name in (
|
|
"api",
|
|
"web",
|
|
"load-balancer",
|
|
"worker",
|
|
"scheduler",
|
|
"ingress",
|
|
)
|
|
if name in service_names(spec)
|
|
]
|
|
_run(
|
|
[*compose, "up", "--detach", "--remove-orphans", *runtime_services],
|
|
cwd=paths.root,
|
|
)
|
|
journal.record(
|
|
"runtime-reconciled",
|
|
"succeeded",
|
|
{"services": runtime_services},
|
|
)
|
|
_wait_for_health(
|
|
f"{spec.public_url}/health/ready",
|
|
timeout_seconds=args.health_timeout_seconds,
|
|
)
|
|
journal.record("health-verified", "succeeded", {"url": spec.public_url})
|
|
receipt = _deployment_receipt(spec, secrets)
|
|
atomic_write(paths.receipt, canonical_json(receipt), mode=0o600)
|
|
journal.succeeded(paths, receipt=receipt)
|
|
except BaseException as exc:
|
|
journal.failed(exc)
|
|
raise
|
|
print(f"GovOPlaN is ready at {spec.public_url}")
|
|
return 0
|
|
|
|
|
|
def _collect_infrastructure_inventory(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
api_key_env = str(args.api_key_env).strip()
|
|
api_key = os.environ.get(api_key_env, "").strip()
|
|
if not api_key:
|
|
raise ValueError(f"{api_key_env} must contain an authorized Ops API key")
|
|
inventory = _collect_dependency_inventory(
|
|
spec,
|
|
paths,
|
|
ops_url=args.ops_url,
|
|
api_key=api_key,
|
|
)
|
|
state = "complete" if inventory.complete else "incomplete"
|
|
print(
|
|
f"Collected {state} provider dependency inventory with "
|
|
f"{len(inventory.dependencies)} record(s) at {paths.dependency_inventory}."
|
|
)
|
|
return 0 if inventory.complete else 1
|
|
|
|
|
|
def _collect_dependency_inventory(
|
|
spec: InstallationSpec,
|
|
paths: BundlePaths,
|
|
*,
|
|
ops_url: str | None,
|
|
api_key: str,
|
|
) -> InfrastructureDependencyInventory:
|
|
url = str(ops_url or "").strip() or (
|
|
spec.public_url.rstrip("/")
|
|
+ "/api/v1/ops/infrastructure/dependencies"
|
|
)
|
|
_validate_ops_inventory_url(url)
|
|
request = Request(
|
|
url,
|
|
headers={"Accept": "application/json", "X-API-Key": api_key},
|
|
)
|
|
with urlopen(request, timeout=15) as response: # noqa: S310
|
|
_validate_ops_inventory_url(response.geturl())
|
|
encoded = response.read(MAX_DEPENDENCY_INVENTORY_BYTES + 1)
|
|
if len(encoded) > MAX_DEPENDENCY_INVENTORY_BYTES:
|
|
raise ValueError("Ops dependency inventory exceeds its size limit")
|
|
value = json.loads(encoded)
|
|
inventory = infrastructure_dependency_inventory_from_mapping(value)
|
|
if inventory.installation_id != spec.installation_id:
|
|
raise ValueError(
|
|
"Ops dependency inventory belongs to a different installation"
|
|
)
|
|
ensure_private_directory(paths.root)
|
|
atomic_write(paths.dependency_inventory, canonical_json(value), mode=0o600)
|
|
return inventory
|
|
|
|
|
|
def _validate_ops_inventory_url(url: str) -> None:
|
|
parsed = urlsplit(url)
|
|
if not parsed.hostname or parsed.username or parsed.password or parsed.fragment:
|
|
raise ValueError("Ops dependency inventory URL is invalid")
|
|
loopback = parsed.hostname in {"localhost", "127.0.0.1", "::1"}
|
|
if parsed.scheme != "https" and not (parsed.scheme == "http" and loopback):
|
|
raise ValueError(
|
|
"Ops dependency inventory URL requires HTTPS except on loopback"
|
|
)
|
|
|
|
|
|
def _status(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
plan = build_plan(spec, paths, include_host_checks=False)
|
|
docker = shutil.which("docker")
|
|
processes: object = []
|
|
process_error = ""
|
|
if docker:
|
|
result = subprocess.run(
|
|
[
|
|
docker,
|
|
"compose",
|
|
"--env-file",
|
|
str(paths.env),
|
|
"--project-name",
|
|
spec.installation_id,
|
|
"--file",
|
|
str(paths.compose),
|
|
"ps",
|
|
"--format",
|
|
"json",
|
|
],
|
|
cwd=paths.root,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
)
|
|
if result.returncode == 0:
|
|
processes = _parse_compose_ps(result.stdout)
|
|
else:
|
|
process_error = result.stderr.strip() or result.stdout.strip()
|
|
else:
|
|
process_error = "Docker CLI is unavailable."
|
|
payload = {
|
|
"installation_id": spec.installation_id,
|
|
"public_url": spec.public_url,
|
|
"plan": plan.to_dict(),
|
|
"processes": processes,
|
|
"process_error": process_error,
|
|
}
|
|
if args.json:
|
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
else:
|
|
_print_plan(plan)
|
|
if process_error:
|
|
print(f"Runtime: {process_error}")
|
|
elif isinstance(processes, list):
|
|
print(f"Runtime: {len(processes)} Compose process(es) reported.")
|
|
return 0
|
|
|
|
|
|
def _verify_release(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
expected_digest = str(args.manifest_sha256 or "").strip().lower()
|
|
if len(expected_digest) != 64 or any(
|
|
character not in "0123456789abcdef" for character in expected_digest
|
|
):
|
|
raise ValueError("--manifest-sha256 must be a lowercase SHA-256 digest")
|
|
manifest_url = str(args.manifest_url or "").strip()
|
|
if manifest_url:
|
|
encoded_manifest = fetch_bounded_https(
|
|
manifest_url,
|
|
maximum_bytes=MAX_MANIFEST_BYTES,
|
|
allow_private_host=args.allow_private_release_host,
|
|
)
|
|
manifest = decode_json_bytes(
|
|
encoded_manifest,
|
|
label="distribution manifest",
|
|
)
|
|
actual_digest = hashlib.sha256(encoded_manifest).hexdigest()
|
|
else:
|
|
manifest_path = args.manifest.expanduser().resolve()
|
|
encoded_manifest = read_bounded_bytes(
|
|
manifest_path,
|
|
maximum_bytes=MAX_MANIFEST_BYTES,
|
|
)
|
|
manifest = load_bounded_json(
|
|
manifest_path,
|
|
maximum_bytes=MAX_MANIFEST_BYTES,
|
|
)
|
|
actual_digest = hashlib.sha256(encoded_manifest).hexdigest()
|
|
if encoded_manifest != canonical_distribution_json(manifest):
|
|
raise DistributionError("distribution manifest is not canonical JSON")
|
|
if actual_digest != expected_digest:
|
|
raise DistributionError("distribution manifest SHA-256 does not match")
|
|
keyring_path = args.trusted_keyring.expanduser().resolve()
|
|
keyring = load_bounded_json(keyring_path, maximum_bytes=MAX_KEYRING_BYTES)
|
|
encoded_keyring = canonical_distribution_json(keyring)
|
|
keyring_digest = hashlib.sha256(encoded_keyring).hexdigest()
|
|
key_id = verify_manifest(
|
|
manifest,
|
|
keyring,
|
|
expected_channel=spec.release.channel,
|
|
)
|
|
dependencies = _selected_dependency_images(spec, manifest=manifest)
|
|
verify_manifest_binding(
|
|
manifest,
|
|
channel=str(manifest["channel"]),
|
|
version=str(manifest["version"]),
|
|
api_image=str(manifest["images"]["api"]["index"]),
|
|
web_image=str(manifest["images"]["web"]["index"]),
|
|
enabled_modules=spec.enabled_modules,
|
|
composition_sha256=str(manifest["composition"]["sha256"]),
|
|
dependencies=dependencies,
|
|
)
|
|
print(
|
|
f"Verified GovOPlaN {manifest['version']} ({manifest['channel']}) "
|
|
f"with trusted key {key_id}."
|
|
)
|
|
if not args.adopt:
|
|
return 0
|
|
|
|
images = manifest["images"]
|
|
dependency_images = manifest["dependencies"]
|
|
release = replace(
|
|
spec.release,
|
|
channel=str(manifest["channel"]),
|
|
version=str(manifest["version"]),
|
|
manifest_url=manifest_url,
|
|
manifest_sha256=expected_digest,
|
|
manifest_keyring_sha256=keyring_digest,
|
|
manifest_signature_key_id=key_id,
|
|
composition_sha256=str(manifest["composition"]["sha256"]),
|
|
api_image=str(images["api"]["index"]),
|
|
web_image=str(images["web"]["index"]),
|
|
)
|
|
components = replace(
|
|
spec.components,
|
|
postgres=replace(
|
|
spec.components.postgres,
|
|
image=(
|
|
str(dependency_images["postgres"])
|
|
if spec.components.postgres.mode == "managed"
|
|
else spec.components.postgres.image
|
|
),
|
|
),
|
|
redis=replace(
|
|
spec.components.redis,
|
|
image=(
|
|
str(dependency_images["redis"])
|
|
if spec.components.redis.mode == "managed"
|
|
else spec.components.redis.image
|
|
),
|
|
),
|
|
mail=replace(
|
|
spec.components.mail,
|
|
image=(
|
|
str(dependency_images["test_mail"])
|
|
if spec.components.mail.mode == "test-mail"
|
|
else spec.components.mail.image
|
|
),
|
|
),
|
|
storage=replace(
|
|
spec.components.storage,
|
|
image=(
|
|
str(dependency_images["garage"])
|
|
if spec.components.storage.mode == "garage"
|
|
else spec.components.storage.image
|
|
),
|
|
),
|
|
load_balancer=replace(
|
|
spec.components.load_balancer,
|
|
image=str(dependency_images["load_balancer"]),
|
|
),
|
|
)
|
|
ingress = replace(
|
|
spec.ingress,
|
|
image=(
|
|
str(dependency_images["managed_ingress"])
|
|
if spec.ingress.mode == "managed"
|
|
else spec.ingress.image
|
|
),
|
|
)
|
|
adopted = parse_spec(
|
|
replace(
|
|
spec,
|
|
release=release,
|
|
components=components,
|
|
ingress=ingress,
|
|
).to_dict()
|
|
)
|
|
ensure_private_directory(paths.root)
|
|
atomic_write(paths.manifest, encoded_manifest, mode=0o644)
|
|
atomic_write(
|
|
paths.keyring,
|
|
encoded_keyring,
|
|
mode=0o644,
|
|
)
|
|
secrets = reconcile_runtime_environment(adopted, read_env(paths.env))
|
|
_write_bundle(adopted, paths, secrets)
|
|
print(f"Adopted immutable runtime distribution in {paths.root}.")
|
|
return 0
|
|
|
|
|
|
def _verify_offline_images(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
manifest = load_bounded_json(
|
|
paths.manifest,
|
|
maximum_bytes=MAX_MANIFEST_BYTES,
|
|
)
|
|
index_path = args.index.expanduser().resolve()
|
|
index = load_bounded_json(
|
|
index_path,
|
|
maximum_bytes=MAX_OFFLINE_INDEX_BYTES,
|
|
)
|
|
selected_dependencies = _selected_dependency_images(spec, manifest=manifest)
|
|
expected = (
|
|
str(manifest["images"]["api"]["index"]),
|
|
str(manifest["images"]["web"]["index"]),
|
|
*tuple(selected_dependencies.values()),
|
|
)
|
|
archives = verify_offline_image_index(
|
|
index,
|
|
root=index_path.parent,
|
|
expected_references=expected,
|
|
)
|
|
print(f"Verified {len(archives)} prefetched OCI image archive(s).")
|
|
if not args.load:
|
|
return 0
|
|
docker = shutil.which("docker")
|
|
if docker is None:
|
|
raise ValueError("Docker CLI is required to load offline images")
|
|
for archive in archives:
|
|
_run([docker, "image", "load", "--input", str(archive)], cwd=paths.root)
|
|
for reference in expected:
|
|
_run([docker, "image", "inspect", reference], cwd=paths.root)
|
|
print("Loaded and inspected every adopted offline image identity.")
|
|
return 0
|
|
|
|
|
|
def _verify_backup(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
ensure_private_directory(paths.root)
|
|
with _deployment_lock(paths.lock):
|
|
return _verify_backup_locked(args, paths)
|
|
|
|
|
|
def _verify_backup_locked(args: argparse.Namespace, paths) -> int:
|
|
spec = load_spec(paths.spec)
|
|
expected_digest = str(args.evidence_sha256 or "").strip().lower()
|
|
if len(expected_digest) != 64 or any(
|
|
character not in "0123456789abcdef" for character in expected_digest
|
|
):
|
|
raise ValueError("--evidence-sha256 must be a lowercase SHA-256 digest")
|
|
if args.evidence_url:
|
|
encoded_evidence = fetch_bounded_https(
|
|
str(args.evidence_url),
|
|
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
|
allow_private_host=args.allow_private_evidence_host,
|
|
)
|
|
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
|
else:
|
|
evidence_path = args.evidence.expanduser().resolve()
|
|
encoded_evidence = read_bounded_bytes(
|
|
evidence_path,
|
|
maximum_bytes=MAX_BACKUP_EVIDENCE_BYTES,
|
|
)
|
|
evidence = decode_json_bytes(encoded_evidence, label="backup evidence")
|
|
if encoded_evidence != canonical_distribution_json(evidence):
|
|
raise DistributionError("backup evidence is not canonical JSON")
|
|
if hashlib.sha256(encoded_evidence).hexdigest() != expected_digest:
|
|
raise DistributionError("backup evidence SHA-256 does not match")
|
|
keyring_path = args.trusted_keyring.expanduser().resolve()
|
|
keyring = load_bounded_json(
|
|
keyring_path,
|
|
maximum_bytes=MAX_BACKUP_KEYRING_BYTES,
|
|
)
|
|
encoded_keyring = canonical_distribution_json(keyring)
|
|
receipt = _read_json_object(paths.receipt)
|
|
previous_release = receipt.get("release") if receipt else None
|
|
release: Mapping[str, object] = (
|
|
previous_release
|
|
if isinstance(previous_release, Mapping)
|
|
else {
|
|
"channel": spec.release.channel,
|
|
"version": spec.release.version,
|
|
"manifest_sha256": spec.release.manifest_sha256,
|
|
"composition_sha256": spec.release.composition_sha256,
|
|
"api_image": spec.release.api_image,
|
|
"web_image": spec.release.web_image,
|
|
}
|
|
)
|
|
summary = verify_backup_evidence(
|
|
evidence,
|
|
keyring,
|
|
installation_id=spec.installation_id,
|
|
profile=spec.profile,
|
|
release=release,
|
|
max_age_seconds=DEFAULT_MAX_BACKUP_AGE_SECONDS,
|
|
)
|
|
print(
|
|
"Verified coordinated recovery point "
|
|
f"{summary['recovery_point_id']} with restore drill "
|
|
f"{summary['restore_drill_id']} and trusted key "
|
|
f"{summary['signature_key_id']}."
|
|
)
|
|
if not args.adopt:
|
|
return 0
|
|
verification = {
|
|
"schema_version": 1,
|
|
"evidence_sha256": expected_digest,
|
|
"keyring_sha256": hashlib.sha256(encoded_keyring).hexdigest(),
|
|
"signature_key_id": summary["signature_key_id"],
|
|
"verified_at": _now(),
|
|
"evidence_id": summary["evidence_id"],
|
|
"recovery_point_id": summary["recovery_point_id"],
|
|
"restore_drill_id": summary["restore_drill_id"],
|
|
"release_manifest_sha256": release.get("manifest_sha256"),
|
|
"captured_at": summary["captured_at"],
|
|
"expires_at": summary["expires_at"],
|
|
"restore_started_at": summary["restore_started_at"],
|
|
"restore_completed_at": summary["restore_completed_at"],
|
|
"measured_rpo_seconds": summary["measured_rpo_seconds"],
|
|
"measured_rto_seconds": summary["measured_rto_seconds"],
|
|
"component_count": summary["component_count"],
|
|
}
|
|
atomic_write(paths.backup_evidence, encoded_evidence, mode=0o600)
|
|
atomic_write(paths.backup_keyring, encoded_keyring, mode=0o600)
|
|
atomic_write(
|
|
paths.backup_verification,
|
|
canonical_json(verification),
|
|
mode=0o600,
|
|
)
|
|
_write_bundle(
|
|
spec,
|
|
paths,
|
|
reconcile_runtime_environment(spec, read_env(paths.env)),
|
|
)
|
|
print(f"Adopted signed backup evidence in {paths.root}.")
|
|
return 0
|
|
|
|
|
|
def _selected_dependency_images(
|
|
spec: InstallationSpec,
|
|
*,
|
|
manifest: Mapping[str, object],
|
|
) -> dict[str, str]:
|
|
available = manifest.get("dependencies")
|
|
if not isinstance(available, dict):
|
|
raise DistributionError("distribution dependencies are invalid")
|
|
names = ["load_balancer"]
|
|
if spec.components.postgres.mode == "managed":
|
|
names.append("postgres")
|
|
if spec.components.redis.mode == "managed":
|
|
names.append("redis")
|
|
if spec.components.mail.mode == "test-mail":
|
|
names.append("test_mail")
|
|
if spec.components.storage.mode == "garage":
|
|
names.append("garage")
|
|
if spec.ingress.mode == "managed":
|
|
names.append("managed_ingress")
|
|
missing = [name for name in names if not isinstance(available.get(name), str)]
|
|
if missing:
|
|
raise DistributionError(
|
|
"distribution is missing selected dependency images: " + ", ".join(missing)
|
|
)
|
|
return {name: str(available[name]) for name in names}
|
|
|
|
|
|
def _render_kubernetes(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
environment = reconcile_runtime_environment(spec, read_env(paths.env))
|
|
environment.update(_backup_runtime_environment(spec, paths))
|
|
receipt = _read_json_object(paths.receipt)
|
|
backup_required = release_change_requires_backup(spec, receipt)
|
|
backup_summary: Mapping[str, object] | None = None
|
|
evidence_files = (
|
|
paths.backup_evidence,
|
|
paths.backup_keyring,
|
|
paths.backup_verification,
|
|
)
|
|
if backup_required:
|
|
backup_summary = verify_stored_backup_evidence(
|
|
spec,
|
|
paths,
|
|
receipt=receipt,
|
|
)
|
|
elif all(path.is_file() for path in evidence_files):
|
|
try:
|
|
backup_summary = verify_stored_backup_evidence(
|
|
spec,
|
|
paths,
|
|
receipt=receipt,
|
|
)
|
|
except (DistributionError, OSError):
|
|
backup_summary = None
|
|
manifest = render_kubernetes(
|
|
spec,
|
|
environment,
|
|
namespace=args.namespace,
|
|
secret_name=args.secret_name,
|
|
tls_secret_name=args.tls_secret_name,
|
|
s3_ca_secret_name=args.s3_ca_secret_name,
|
|
ingress_class_name=args.ingress_class_name,
|
|
backup_required=backup_required,
|
|
backup_evidence=backup_summary,
|
|
)
|
|
output = (args.output or (paths.root / "kubernetes.json")).expanduser().resolve()
|
|
atomic_write(output, canonical_json(manifest), mode=0o600)
|
|
print(f"Wrote stateless Kubernetes runtime manifest to {output}")
|
|
print("Required Secret keys: " + ", ".join(kubernetes_secret_contract()))
|
|
print(
|
|
write_secret_creation_hint(
|
|
paths.env,
|
|
namespace=args.namespace,
|
|
secret_name=args.secret_name,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
def _verify_kubernetes(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
spec = load_spec(paths.spec)
|
|
api_key = str(os.environ.get(args.api_key_env) or "").strip()
|
|
if not api_key:
|
|
raise ValueError(
|
|
f"{args.api_key_env} must contain an API key authorized to read "
|
|
"Ops status"
|
|
)
|
|
ops_url = args.ops_url or (spec.public_url.rstrip("/") + "/api/v1/ops/status")
|
|
evidence = collect_kubernetes_evidence(
|
|
installation_id=spec.installation_id,
|
|
namespace=args.namespace,
|
|
ops_url=ops_url,
|
|
api_key=api_key,
|
|
exercise_api_pod_loss=args.exercise_api_pod_loss,
|
|
timeout_seconds=args.timeout_seconds,
|
|
)
|
|
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
output = (
|
|
(args.output or paths.root / "evidence" / f"kubernetes-{timestamp}.json")
|
|
.expanduser()
|
|
.resolve()
|
|
)
|
|
ensure_private_directory(output.parent)
|
|
atomic_write(output, canonical_json(evidence), mode=0o600)
|
|
print(f"Wrote Kubernetes evidence to {output}")
|
|
if evidence["result"]["state"] != "passed":
|
|
print(
|
|
"Failed checks: " + ", ".join(evidence["result"]["failed_checks"]),
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def _operations(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
payload = list_operations(paths)
|
|
if args.json:
|
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
elif not payload:
|
|
print("No deployment operations recorded.")
|
|
else:
|
|
for operation in payload:
|
|
print(
|
|
f"{operation['operation_id']} {operation['status']} "
|
|
f"{operation['recovery_mode']} stages={operation['stage_count']}"
|
|
)
|
|
if operation.get("failure_summary"):
|
|
print(f" {operation['failure_summary']}")
|
|
return 0
|
|
|
|
|
|
def _recover(args: argparse.Namespace) -> int:
|
|
paths = bundle_paths(args.directory)
|
|
ensure_private_directory(paths.root)
|
|
with _deployment_lock(paths.lock):
|
|
result = recover_operation(paths, operation_id=args.operation_id)
|
|
print(f"Recovery operation {result.operation_id}: {result.action}")
|
|
print(result.detail)
|
|
if args.apply:
|
|
if not result.apply_allowed:
|
|
raise ValueError("this recovery state cannot be applied automatically")
|
|
return _apply(args)
|
|
if result.apply_allowed:
|
|
print(f"Reconcile with: govoplan-deploy apply --directory {paths.root}")
|
|
return 0
|
|
|
|
|
|
def _deployment_receipt(
|
|
spec: InstallationSpec,
|
|
secrets: Mapping[str, str],
|
|
) -> dict[str, object]:
|
|
return {
|
|
"schema_version": 1,
|
|
"installation_id": spec.installation_id,
|
|
"profile": spec.profile,
|
|
"applied_at": _now(),
|
|
"spec_sha256": digest_json(spec.to_dict()),
|
|
"compose_sha256": digest_json(render_compose(spec)),
|
|
"environment_fingerprint": environment_fingerprint(secrets),
|
|
"release": {
|
|
"channel": spec.release.channel,
|
|
"version": spec.release.version,
|
|
"manifest_sha256": spec.release.manifest_sha256,
|
|
"manifest_keyring_sha256": spec.release.manifest_keyring_sha256,
|
|
"manifest_signature_key_id": spec.release.manifest_signature_key_id,
|
|
"composition_sha256": spec.release.composition_sha256,
|
|
"api_image": spec.release.api_image,
|
|
"web_image": spec.release.web_image,
|
|
},
|
|
"services": list(service_names(spec)),
|
|
"replicas": {
|
|
"api": spec.replicas.api,
|
|
"web": spec.replicas.web,
|
|
"worker": spec.replicas.worker,
|
|
},
|
|
"listen": {
|
|
"address": spec.listen.address,
|
|
"port": spec.listen.port,
|
|
},
|
|
"ingress": {
|
|
"mode": spec.ingress.mode,
|
|
"http_port": spec.ingress.http_port,
|
|
"https_port": spec.ingress.https_port,
|
|
},
|
|
"management": {
|
|
"mode": "govoplan-deploy",
|
|
"agent": "cli",
|
|
"web_updates": False,
|
|
},
|
|
"infrastructure_capabilities": infrastructure_capability_document(
|
|
spec,
|
|
secrets,
|
|
),
|
|
}
|
|
|
|
|
|
def _read_json_object(path: Path) -> dict[str, object]:
|
|
if not path.is_file():
|
|
return {}
|
|
try:
|
|
value = load_bounded_json(path, maximum_bytes=64 * 1024)
|
|
except (DistributionError, OSError):
|
|
return {}
|
|
return value
|
|
|
|
|
|
def _updated_spec(
|
|
current: InstallationSpec, args: argparse.Namespace
|
|
) -> InstallationSpec:
|
|
module_set = getattr(args, "module_set", None)
|
|
if module_set:
|
|
modules = default_spec(module_set=module_set).enabled_modules
|
|
else:
|
|
modules = current.enabled_modules
|
|
release = replace(
|
|
current.release,
|
|
channel=args.release_channel or current.release.channel,
|
|
version=args.release_version or current.release.version,
|
|
manifest_url=(
|
|
args.manifest_url
|
|
if args.manifest_url is not None
|
|
else current.release.manifest_url
|
|
),
|
|
manifest_sha256=(
|
|
args.manifest_sha256
|
|
if args.manifest_sha256 is not None
|
|
else current.release.manifest_sha256
|
|
),
|
|
manifest_keyring_sha256=current.release.manifest_keyring_sha256,
|
|
manifest_signature_key_id=current.release.manifest_signature_key_id,
|
|
composition_sha256=current.release.composition_sha256,
|
|
api_image=args.api_image or current.release.api_image,
|
|
web_image=args.web_image or current.release.web_image,
|
|
)
|
|
storage_mode = args.storage or current.components.storage.mode
|
|
components = ComponentConfig(
|
|
postgres=replace(
|
|
current.components.postgres,
|
|
mode=args.postgres or current.components.postgres.mode,
|
|
),
|
|
redis=replace(
|
|
current.components.redis,
|
|
mode=args.redis or current.components.redis.mode,
|
|
),
|
|
mail=replace(
|
|
current.components.mail,
|
|
mode=args.mail or current.components.mail.mode,
|
|
),
|
|
storage=replace(
|
|
current.components.storage,
|
|
mode=storage_mode,
|
|
image=(
|
|
args.garage_image
|
|
or current.components.storage.image
|
|
or DEFAULT_GARAGE_IMAGE
|
|
)
|
|
if storage_mode == "garage"
|
|
else "",
|
|
),
|
|
load_balancer=replace(
|
|
current.components.load_balancer,
|
|
image=(args.load_balancer_image or current.components.load_balancer.image),
|
|
),
|
|
)
|
|
replicas = ReplicaConfig(
|
|
api=(
|
|
args.api_replicas if args.api_replicas is not None else current.replicas.api
|
|
),
|
|
web=(
|
|
args.web_replicas if args.web_replicas is not None else current.replicas.web
|
|
),
|
|
worker=(
|
|
args.worker_replicas
|
|
if args.worker_replicas is not None
|
|
else (
|
|
0
|
|
if components.redis.mode == "disabled"
|
|
else max(current.replicas.worker, 1)
|
|
)
|
|
),
|
|
)
|
|
ingress_mode = args.ingress or current.ingress.mode
|
|
ingress = IngressConfig(
|
|
mode=ingress_mode,
|
|
image=(args.ingress_image or current.ingress.image or DEFAULT_INGRESS_IMAGE)
|
|
if ingress_mode == "managed"
|
|
else "",
|
|
trusted_proxy_cidrs=tuple(
|
|
(
|
|
args.trusted_proxy_cidr
|
|
if args.trusted_proxy_cidr is not None
|
|
else current.ingress.trusted_proxy_cidrs
|
|
)
|
|
if ingress_mode == "existing-proxy"
|
|
else ()
|
|
),
|
|
http_port=(
|
|
args.ingress_http_port
|
|
if args.ingress_http_port is not None
|
|
else current.ingress.http_port
|
|
),
|
|
https_port=(
|
|
args.ingress_https_port
|
|
if args.ingress_https_port is not None
|
|
else current.ingress.https_port
|
|
),
|
|
acme_email=(
|
|
(
|
|
args.acme_email
|
|
if args.acme_email is not None
|
|
else current.ingress.acme_email
|
|
)
|
|
if ingress_mode == "managed"
|
|
else ""
|
|
),
|
|
)
|
|
value = replace(
|
|
current,
|
|
installation_id=args.installation_id or current.installation_id,
|
|
profile=args.profile or current.profile,
|
|
public_url=args.public_url or current.public_url,
|
|
listen=ListenConfig(
|
|
address=args.listen_address or current.listen.address,
|
|
port=args.listen_port or current.listen.port,
|
|
),
|
|
release=release,
|
|
components=components,
|
|
replicas=replicas,
|
|
ingress=ingress,
|
|
enabled_modules=modules,
|
|
)
|
|
return parse_spec(value.to_dict())
|
|
|
|
|
|
def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]:
|
|
values: dict[str, str] = {}
|
|
if getattr(args, "database_url", None):
|
|
values["DATABASE_URL"] = args.database_url
|
|
values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = _pgtools_url(args.database_url)
|
|
if getattr(args, "redis_url", None):
|
|
values["REDIS_URL"] = args.redis_url
|
|
s3_values = {
|
|
"FILE_STORAGE_S3_ENDPOINT_URL": getattr(args, "s3_endpoint_url", None),
|
|
"FILE_STORAGE_S3_REGION": getattr(args, "s3_region", None),
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(args, "s3_access_key_id", None),
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": getattr(
|
|
args, "s3_secret_access_key", None
|
|
),
|
|
"FILE_STORAGE_S3_BUCKET": getattr(args, "s3_bucket", None),
|
|
}
|
|
values.update({key: value for key, value in s3_values.items() if value})
|
|
return values
|
|
|
|
|
|
def _pgtools_url(database_url: str) -> str:
|
|
if database_url.startswith("postgresql+psycopg://"):
|
|
return "postgresql://" + database_url.removeprefix("postgresql+psycopg://")
|
|
return database_url
|
|
|
|
|
|
def _write_bundle(
|
|
spec: InstallationSpec,
|
|
paths,
|
|
secrets: Mapping[str, str],
|
|
) -> dict[str, str]:
|
|
ensure_private_directory(paths.root)
|
|
runtime_environment = dict(secrets)
|
|
runtime_environment.update(_backup_runtime_environment(spec, paths))
|
|
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
|
|
write_env(paths.env, runtime_environment)
|
|
atomic_write(
|
|
paths.capabilities,
|
|
canonical_json(infrastructure_capability_document(spec, runtime_environment)),
|
|
mode=0o644,
|
|
)
|
|
atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
|
|
atomic_write(
|
|
paths.load_balancer_config,
|
|
render_load_balancer_config(spec).encode("utf-8"),
|
|
mode=0o644,
|
|
)
|
|
atomic_write(
|
|
paths.caddy_config,
|
|
render_caddy_config(spec).encode("utf-8"),
|
|
mode=0o644,
|
|
)
|
|
atomic_write(
|
|
paths.existing_proxy,
|
|
canonical_json(render_existing_proxy_contract(spec)),
|
|
mode=0o644,
|
|
)
|
|
atomic_write(
|
|
paths.garage_config,
|
|
render_garage_config().encode("utf-8"),
|
|
mode=0o644,
|
|
)
|
|
return dict(sorted(runtime_environment.items()))
|
|
|
|
|
|
def _backup_runtime_environment(
|
|
spec: InstallationSpec,
|
|
paths,
|
|
) -> dict[str, str]:
|
|
values = {key: "" for key in BACKUP_RUNTIME_ENV_KEYS}
|
|
evidence_files = (
|
|
paths.backup_evidence,
|
|
paths.backup_keyring,
|
|
paths.backup_verification,
|
|
)
|
|
if not any(path.is_file() for path in evidence_files):
|
|
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "absent"
|
|
return values
|
|
if not all(path.is_file() for path in evidence_files):
|
|
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
|
|
return values
|
|
verification = _read_json_object(paths.backup_verification)
|
|
try:
|
|
summary = verify_stored_backup_evidence(
|
|
spec,
|
|
paths,
|
|
receipt=_read_json_object(paths.receipt),
|
|
)
|
|
except (DistributionError, OSError):
|
|
values["GOVOPLAN_BACKUP_EVIDENCE_STATE"] = "invalid"
|
|
return values
|
|
values.update(
|
|
{
|
|
"GOVOPLAN_BACKUP_EVIDENCE_STATE": "verified",
|
|
"GOVOPLAN_BACKUP_EVIDENCE_ID": str(summary["evidence_id"]),
|
|
"GOVOPLAN_BACKUP_RECOVERY_POINT_ID": str(summary["recovery_point_id"]),
|
|
"GOVOPLAN_BACKUP_RESTORE_DRILL_ID": str(summary["restore_drill_id"]),
|
|
"GOVOPLAN_BACKUP_EVIDENCE_SHA256": str(summary["evidence_sha256"]),
|
|
"GOVOPLAN_BACKUP_RELEASE_MANIFEST_SHA256": str(
|
|
verification["release_manifest_sha256"]
|
|
),
|
|
"GOVOPLAN_BACKUP_CAPTURED_AT": str(summary["captured_at"]),
|
|
"GOVOPLAN_BACKUP_EXPIRES_AT": str(summary["expires_at"]),
|
|
"GOVOPLAN_BACKUP_RESTORE_STARTED_AT": str(summary["restore_started_at"]),
|
|
"GOVOPLAN_BACKUP_RESTORE_COMPLETED_AT": str(
|
|
summary["restore_completed_at"]
|
|
),
|
|
"GOVOPLAN_BACKUP_VERIFIED_AT": str(verification["verified_at"]),
|
|
"GOVOPLAN_BACKUP_MEASURED_RPO_SECONDS": str(
|
|
summary["measured_rpo_seconds"]
|
|
),
|
|
"GOVOPLAN_BACKUP_MEASURED_RTO_SECONDS": str(
|
|
summary["measured_rto_seconds"]
|
|
),
|
|
"GOVOPLAN_BACKUP_COMPONENT_COUNT": str(summary["component_count"]),
|
|
}
|
|
)
|
|
return values
|
|
|
|
|
|
def _write_plan(path: Path, plan: DeploymentPlan) -> None:
|
|
atomic_write(path, canonical_json(plan.to_dict()), mode=0o600)
|
|
|
|
|
|
def _print_plan(plan: DeploymentPlan) -> None:
|
|
state = "BLOCKED" if plan.blocked else "READY"
|
|
print(f"GovOPlaN deployment plan: {state}")
|
|
for action in plan.actions:
|
|
print(f" {action.action:12} {action.target}: {action.detail}")
|
|
for check in plan.checks:
|
|
prefix = {"ok": "OK", "warning": "WARN", "error": "ERROR"}.get(
|
|
check.level, check.level.upper()
|
|
)
|
|
print(f" [{prefix}] {check.message}")
|
|
if check.action and check.level != "ok":
|
|
print(f" {check.action}")
|
|
|
|
|
|
def _prompt_configuration(args: argparse.Namespace) -> None:
|
|
args.profile = _prompt_choice(
|
|
"Installation profile", args.profile, ("evaluation", "self-hosted")
|
|
)
|
|
if args.profile == "self-hosted" and args.public_url.startswith("http://"):
|
|
args.public_url = "https://govoplan.example.org"
|
|
args.public_url = _prompt("Public URL", args.public_url)
|
|
if args.profile == "self-hosted":
|
|
args.ingress = _prompt_choice(
|
|
"Public ingress",
|
|
args.ingress or "existing-proxy",
|
|
("existing-proxy", "managed"),
|
|
)
|
|
if args.ingress == "existing-proxy":
|
|
current = (args.trusted_proxy_cidr or ["127.0.0.1/32"])[0]
|
|
args.trusted_proxy_cidr = [
|
|
_prompt("Trusted reverse-proxy source CIDR", current)
|
|
]
|
|
else:
|
|
args.acme_email = args.acme_email or _prompt(
|
|
"ACME account email",
|
|
"admin@example.org",
|
|
)
|
|
else:
|
|
args.ingress = args.ingress or "local"
|
|
args.postgres = _prompt_choice("PostgreSQL", args.postgres, ("managed", "external"))
|
|
if args.postgres == "external" and not args.database_url:
|
|
args.database_url = getpass.getpass(
|
|
"External PostgreSQL URL (input hidden): "
|
|
).strip()
|
|
redis_choices = (
|
|
("managed", "external")
|
|
if args.profile == "self-hosted"
|
|
else ("managed", "external", "disabled")
|
|
)
|
|
args.redis = _prompt_choice("Redis", args.redis, redis_choices)
|
|
if args.redis == "external" and not args.redis_url:
|
|
args.redis_url = getpass.getpass("External Redis URL (input hidden): ").strip()
|
|
mail_choices = (
|
|
("disabled", "external-relay")
|
|
if args.profile == "self-hosted"
|
|
else ("disabled", "external-relay", "test-mail")
|
|
)
|
|
args.mail = _prompt_choice(
|
|
"Mail integration",
|
|
args.mail,
|
|
mail_choices,
|
|
)
|
|
args.storage = _prompt_choice(
|
|
"File storage",
|
|
args.storage,
|
|
("local", "garage", "s3"),
|
|
)
|
|
if args.storage == "s3":
|
|
args.s3_endpoint_url = args.s3_endpoint_url or _prompt(
|
|
"S3 endpoint URL", "https://s3.example.org"
|
|
)
|
|
args.s3_region = args.s3_region or _prompt("S3 region", "eu-central-1")
|
|
args.s3_access_key_id = args.s3_access_key_id or _prompt("S3 access key id", "")
|
|
args.s3_secret_access_key = (
|
|
args.s3_secret_access_key
|
|
or getpass.getpass("S3 secret access key (input hidden): ").strip()
|
|
)
|
|
args.s3_bucket = args.s3_bucket or _prompt("S3 bucket", "govoplan-files")
|
|
args.api_replicas = _prompt_integer(
|
|
"API replicas on this host",
|
|
args.api_replicas,
|
|
minimum=1,
|
|
maximum=64,
|
|
)
|
|
args.web_replicas = _prompt_integer(
|
|
"WebUI replicas on this host",
|
|
args.web_replicas,
|
|
minimum=1,
|
|
maximum=64,
|
|
)
|
|
if args.redis != "disabled":
|
|
args.worker_replicas = _prompt_integer(
|
|
"Worker replicas on this host",
|
|
args.worker_replicas or 1,
|
|
minimum=1,
|
|
maximum=128,
|
|
)
|
|
else:
|
|
args.worker_replicas = 0
|
|
args.module_set = _prompt_choice(
|
|
"Initial module set", args.module_set, ("core", "base", "full")
|
|
)
|
|
|
|
|
|
def _prompt(label: str, default: str) -> str:
|
|
value = input(f"{label} [{default}]: ").strip()
|
|
return value or default
|
|
|
|
|
|
def _prompt_choice(label: str, default: str, choices: tuple[str, ...]) -> str:
|
|
while True:
|
|
value = _prompt(f"{label} ({'/'.join(choices)})", default)
|
|
if value in choices:
|
|
return value
|
|
print(f"Choose one of: {', '.join(choices)}")
|
|
|
|
|
|
def _prompt_integer(
|
|
label: str,
|
|
default: int,
|
|
*,
|
|
minimum: int,
|
|
maximum: int,
|
|
) -> int:
|
|
while True:
|
|
raw = _prompt(label, str(default))
|
|
try:
|
|
value = int(raw)
|
|
except ValueError:
|
|
value = minimum - 1
|
|
if minimum <= value <= maximum:
|
|
return value
|
|
print(f"Choose a number between {minimum} and {maximum}.")
|
|
|
|
|
|
@contextmanager
|
|
def _deployment_lock(path: Path) -> Iterator[None]:
|
|
descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
try:
|
|
try:
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise ValueError(
|
|
"another deployment operation owns the installation lock"
|
|
) from exc
|
|
os.ftruncate(descriptor, 0)
|
|
os.write(descriptor, f"{os.getpid()}\n".encode("ascii"))
|
|
os.fsync(descriptor)
|
|
yield
|
|
finally:
|
|
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
os.close(descriptor)
|
|
|
|
|
|
def _run(argv: Sequence[str], *, cwd: Path) -> None:
|
|
result = subprocess.run(list(argv), cwd=cwd, check=False)
|
|
if result.returncode != 0:
|
|
raise subprocess.CalledProcessError(result.returncode, list(argv))
|
|
|
|
|
|
def _wait_for_health(url: str, *, timeout_seconds: float) -> None:
|
|
deadline = time.monotonic() + max(timeout_seconds, 1.0)
|
|
last_error = "health endpoint did not answer"
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urlopen(url, timeout=3) as response: # noqa: S310 - URL originates in the validated installation specification.
|
|
if 200 <= response.status < 300:
|
|
return
|
|
last_error = f"health endpoint returned HTTP {response.status}"
|
|
except (OSError, URLError) as exc:
|
|
last_error = str(exc)
|
|
time.sleep(2)
|
|
raise ValueError(f"GovOPlaN did not become healthy: {last_error}")
|
|
|
|
|
|
def _parse_compose_ps(output: str) -> list[object]:
|
|
stripped = output.strip()
|
|
if not stripped:
|
|
return []
|
|
try:
|
|
value = json.loads(stripped)
|
|
except json.JSONDecodeError:
|
|
rows = []
|
|
for line in stripped.splitlines():
|
|
try:
|
|
rows.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
return [{"raw": stripped}]
|
|
return rows
|
|
return value if isinstance(value, list) else [value]
|
|
|
|
|
|
def _receipt_uses_direct_web_port(path: Path) -> bool:
|
|
if not path.exists():
|
|
return False
|
|
try:
|
|
receipt = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return False
|
|
services = receipt.get("services") if isinstance(receipt, dict) else None
|
|
return (
|
|
isinstance(services, list)
|
|
and "web" in services
|
|
and "load-balancer" not in services
|
|
)
|
|
|
|
|
|
def _now() -> str:
|
|
return (
|
|
datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
)
|