"""Secret handling and deterministic Compose bundle rendering.""" from __future__ import annotations import base64 from dataclasses import dataclass from hashlib import sha256 import hmac import json import os from pathlib import Path import secrets import stat from typing import Mapping from urllib.parse import quote, urlsplit from .model import ENV_NAME_PATTERN, InstallationSpec SPEC_FILENAME = "installation.json" ENV_FILENAME = "secrets.env" COMPOSE_FILENAME = "compose.json" PLAN_FILENAME = "plan.json" RECEIPT_FILENAME = "receipt.json" LOCK_FILENAME = ".deployment.lock" RUNTIME_ENV_KEYS = ( "APP_ENV", "GOVOPLAN_INSTALL_PROFILE", "MASTER_KEY_B64", "DATABASE_URL", "GOVOPLAN_DATABASE_URL_PGTOOLS", "ENABLED_MODULES", "CELERY_ENABLED", "CELERY_QUEUES", "REDIS_URL", "CORS_ORIGINS", "GOVOPLAN_TRUSTED_HOSTS", "FORWARDED_ALLOW_IPS", "AUTH_COOKIE_SECURE", "AUTH_COOKIE_SAMESITE", "GOVOPLAN_HTTP_HSTS_SECONDS", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", "GOVOPLAN_MIGRATION_TRACK", "DEV_AUTO_MIGRATE_ENABLED", "DEV_BOOTSTRAP_ENABLED", "GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE", "GOVOPLAN_DEPLOYMENT_SPEC_PATH", "FILE_STORAGE_BACKEND", "FILE_STORAGE_LOCAL_ROOT", "FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET", ) @dataclass(frozen=True, slots=True) class BundlePaths: root: Path spec: Path env: Path compose: Path plan: Path receipt: Path lock: Path def bundle_paths(root: Path) -> BundlePaths: expanded = root.expanduser() if expanded.is_symlink(): raise ValueError( f"installation root must not be a symbolic link: {expanded}" ) resolved = expanded.resolve() return BundlePaths( root=resolved, spec=resolved / SPEC_FILENAME, env=resolved / ENV_FILENAME, compose=resolved / COMPOSE_FILENAME, plan=resolved / PLAN_FILENAME, receipt=resolved / RECEIPT_FILENAME, lock=resolved / LOCK_FILENAME, ) def ensure_private_directory(path: Path) -> None: path.mkdir(mode=0o700, parents=True, exist_ok=True) if path.is_symlink() or not path.is_dir(): raise ValueError(f"installation root must be a real directory: {path}") current = stat.S_IMODE(path.stat().st_mode) if current & 0o077: path.chmod(0o700) def initial_secrets( spec: InstallationSpec, *, supplied: Mapping[str, str] | None = None, ) -> dict[str, str]: values = dict(supplied or {}) values.setdefault( "MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"), ) values.setdefault("POSTGRES_DB", "govoplan") values.setdefault("POSTGRES_USER", "govoplan") values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) return reconcile_runtime_environment(spec, values) def reconcile_runtime_environment( spec: InstallationSpec, current: Mapping[str, str], ) -> dict[str, str]: values = dict(current) postgres = spec.components.postgres if postgres.mode == "managed": database = values.setdefault("POSTGRES_DB", "govoplan") username = values.setdefault("POSTGRES_USER", "govoplan") password = values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) encoded_user = quote(username, safe="") encoded_password = quote(password, safe="") encoded_database = quote(database, safe="") values["DATABASE_URL"] = ( f"postgresql+psycopg://{encoded_user}:{encoded_password}" f"@postgres:5432/{encoded_database}" ) values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = ( f"postgresql://{encoded_user}:{encoded_password}" f"@postgres:5432/{encoded_database}" ) elif not values.get(postgres.url_env): raise ValueError( f"external PostgreSQL requires {postgres.url_env} in {ENV_FILENAME}" ) else: _validate_service_url( values[postgres.url_env], schemes={"postgresql", "postgresql+psycopg"}, label="external PostgreSQL URL", ) redis = spec.components.redis if redis.mode == "managed": password = values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) values["REDIS_URL"] = f"redis://:{quote(password, safe='')}@redis:6379/0" elif redis.mode == "external": if not values.get(redis.url_env): raise ValueError( f"external Redis requires {redis.url_env} in {ENV_FILENAME}" ) _validate_service_url( values[redis.url_env], schemes={"redis", "rediss"}, label="external Redis URL", ) else: values["REDIS_URL"] = "" public = urlsplit(spec.public_url) values.update( { "APP_ENV": "production" if spec.profile == "self-hosted" else "staging", "GOVOPLAN_INSTALL_PROFILE": spec.profile, "ENABLED_MODULES": ",".join(spec.enabled_modules), "CELERY_ENABLED": "true" if redis.mode != "disabled" else "false", "CELERY_QUEUES": ( "send_email,append_sent,notifications,calendar,dataflow,events,default" ), "CORS_ORIGINS": spec.public_url, "GOVOPLAN_TRUSTED_HOSTS": public.hostname or "", "FORWARDED_ALLOW_IPS": spec.network_subnet, "AUTH_COOKIE_SECURE": "true" if public.scheme == "https" else "false", "AUTH_COOKIE_SAMESITE": "lax", "GOVOPLAN_HTTP_HSTS_SECONDS": ( "31536000" if public.scheme == "https" else "0" ), "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false", "GOVOPLAN_MIGRATION_TRACK": "release", "DEV_AUTO_MIGRATE_ENABLED": "false", "DEV_BOOTSTRAP_ENABLED": "false", "GOVOPLAN_DEPLOYMENT_SPEC_PATH": "/etc/govoplan/deployment/installation.json", } ) if redis.mode == "disabled": values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true" else: values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false" storage = spec.components.storage values["FILE_STORAGE_BACKEND"] = storage.mode if storage.mode == "local": values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files" else: required = ( "FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET", ) missing = [name for name in required if not values.get(name)] if missing: raise ValueError( "S3 storage requires values in secrets.env: " + ", ".join(missing) ) endpoint = _validate_service_url( values["FILE_STORAGE_S3_ENDPOINT_URL"], schemes={"http", "https"}, label="S3 endpoint URL", ) if spec.profile == "self-hosted" and endpoint.scheme != "https": raise ValueError("self-hosted S3 endpoint URL must use HTTPS") return dict(sorted(values.items())) def render_compose(spec: InstallationSpec) -> dict[str, object]: runtime_env = { "environment": _environment_references(RUNTIME_ENV_KEYS), } deployment_mount = ( f"./{SPEC_FILENAME}:/etc/govoplan/deployment/installation.json:ro" ) data_mounts = [deployment_mount] if spec.components.storage.mode == "local": data_mounts.append("files-data:/var/lib/govoplan/files") dependency_conditions: dict[str, dict[str, str]] = {} services: dict[str, object] = {} if spec.components.postgres.mode == "managed": services["postgres"] = { "image": spec.components.postgres.image, "restart": "unless-stopped", "environment": _environment_references( ("POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"), required=True, ), "healthcheck": { "test": [ "CMD-SHELL", 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"', ], "interval": "5s", "timeout": "3s", "retries": 30, }, "volumes": ["postgres-data:/var/lib/postgresql/data"], "networks": ["internal"], } dependency_conditions["postgres"] = {"condition": "service_healthy"} if spec.components.redis.mode == "managed": services["redis"] = { "image": spec.components.redis.image, "restart": "unless-stopped", "environment": _environment_references( ("REDIS_PASSWORD",), required=True, ), "command": [ "sh", "-ec", 'exec redis-server --appendonly yes --requirepass "$$REDIS_PASSWORD"', ], "healthcheck": { "test": [ "CMD-SHELL", 'redis-cli -a "$${REDIS_PASSWORD}" --no-auth-warning ping', ], "interval": "5s", "timeout": "3s", "retries": 30, }, "volumes": ["redis-data:/data"], "networks": ["internal"], } dependency_conditions["redis"] = {"condition": "service_healthy"} if spec.components.mail.mode == "test-mail": services["test-mail"] = { "image": spec.components.mail.image, "restart": "unless-stopped", "environment": { "GREENMAIL_OPTS": ( "-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap " "-Dgreenmail.hostname=0.0.0.0" ) }, "networks": ["internal"], } common_runtime: dict[str, object] = { **runtime_env, "restart": "unless-stopped", "volumes": data_mounts, "networks": ["internal"], } if dependency_conditions: common_runtime["depends_on"] = dependency_conditions services["migrate"] = { **runtime_env, "image": spec.release.api_image, "command": ["python", "-m", "govoplan_core.commands.init_db"], "restart": "no", "volumes": data_mounts, "networks": ["internal"], **({"depends_on": dependency_conditions} if dependency_conditions else {}), } services["api"] = { **common_runtime, "image": spec.release.api_image, "command": [ "python", "-m", "uvicorn", "govoplan_core.server.app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", ], "healthcheck": { "test": [ "CMD", "python", "-c", ( "import urllib.request;" "urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)" ), ], "interval": "10s", "timeout": "5s", "retries": 30, "start_period": "20s", }, } services["web"] = { "image": spec.release.web_image, "restart": "unless-stopped", "environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"}, "depends_on": {"api": {"condition": "service_healthy"}}, "ports": [_published_port(spec.listen.address, spec.listen.port, 8080)], "networks": ["internal"], } if spec.components.redis.mode != "disabled": services["worker"] = { **common_runtime, "image": spec.release.api_image, "command": [ "python", "-m", "celery", "-A", "govoplan_core.celery_app:celery", "worker", "--queues", ( "send_email,append_sent,notifications,calendar," "dataflow,events,default" ), "--loglevel", "INFO", ], } services["scheduler"] = { **common_runtime, "image": spec.release.api_image, "command": [ "python", "-m", "celery", "-A", "govoplan_core.celery_app:celery", "beat", "--loglevel", "INFO", ], } volumes: dict[str, object] = {} if spec.components.postgres.mode == "managed": volumes["postgres-data"] = {} if spec.components.redis.mode == "managed": volumes["redis-data"] = {} if spec.components.storage.mode == "local": volumes["files-data"] = {} return { "name": spec.installation_id, "services": services, "volumes": volumes, "networks": { "internal": { "driver": "bridge", "ipam": {"config": [{"subnet": spec.network_subnet}]}, } }, } def service_names(spec: InstallationSpec) -> tuple[str, ...]: return tuple(render_compose(spec)["services"].keys()) def canonical_json(value: object) -> bytes: return ( json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + "\n" ).encode("utf-8") def digest_json(value: object) -> str: return sha256(canonical_json(value)).hexdigest() def environment_fingerprint(values: Mapping[str, str]) -> str: key = values.get("MASTER_KEY_B64", "").encode("utf-8") if not key: return "" payload = canonical_json(dict(sorted(values.items()))) return hmac.new(key, payload, sha256).hexdigest() def read_env(path: Path) -> dict[str, str]: if not path.exists(): return {} values: dict[str, str] = {} for line_number, raw_line in enumerate( path.read_text(encoding="utf-8").splitlines(), start=1 ): line = raw_line.strip() if not line or line.startswith("#"): continue key, separator, raw_value = line.partition("=") if not separator or not ENV_NAME_PATTERN.fullmatch(key): raise ValueError(f"invalid environment line {line_number} in {path}") if key in values: raise ValueError( f"duplicate environment key {key!r} on line {line_number}" ) value = raw_value if value.startswith('"'): try: decoded = json.loads(value) except json.JSONDecodeError as exc: raise ValueError( f"invalid quoted environment value on line {line_number}" ) from exc if not isinstance(decoded, str): raise ValueError( f"environment value on line {line_number} must be a string" ) value = decoded elif value.startswith("'"): value = _single_quoted_env_value(value, line_number=line_number) values[key] = value return values def write_env(path: Path, values: Mapping[str, str]) -> None: invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key)) if invalid: raise ValueError( "invalid environment variable names: " + ", ".join(invalid) ) lines = [ "# Generated by govoplan-deploy. Keep this file private.", *[f"{key}={_env_value(value)}" for key, value in sorted(values.items())], "", ] atomic_write(path, "\n".join(lines).encode("utf-8"), mode=0o600) def atomic_write(path: Path, payload: bytes, *, mode: int) -> None: path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) temporary = path.with_name( f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" ) descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode, ) try: with os.fdopen(descriptor, "wb", closefd=True) as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) path.chmod(mode) directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary.exists(): temporary.unlink() def _env_value(value: str) -> str: if "\x00" in value or "\n" in value or "\r" in value: raise ValueError("environment values must not contain NUL or line breaks") if value and all( character.isalnum() or character in "_./:@%+,-" for character in value ): return value escaped = value.replace("\\", "\\\\").replace("'", "\\'") return f"'{escaped}'" def _validate_service_url( value: str, *, schemes: set[str], label: str, ): parsed = urlsplit(value) try: parsed.port except ValueError as exc: raise ValueError(f"{label} contains an invalid port") from exc if parsed.scheme not in schemes or not parsed.hostname: raise ValueError( f"{label} must use one of {', '.join(sorted(schemes))} and include a host" ) if parsed.fragment: raise ValueError(f"{label} must not contain a fragment") return parsed def _single_quoted_env_value(value: str, *, line_number: int) -> str: if len(value) < 2 or not value.endswith("'"): raise ValueError( f"unterminated single-quoted environment value on line {line_number}" ) body = value[1:-1] output: list[str] = [] index = 0 while index < len(body): character = body[index] if character != "\\": output.append(character) index += 1 continue index += 1 if index >= len(body) or body[index] not in {"\\", "'"}: raise ValueError( f"invalid single-quoted escape on line {line_number}" ) output.append(body[index]) index += 1 return "".join(output) def _published_port(address: str, host_port: int, container_port: int) -> str: host = f"[{address}]" if ":" in address else address return f"{host}:{host_port}:{container_port}" def _environment_references( keys: tuple[str, ...], *, required: bool = False, ) -> dict[str, str]: suffix = ":?required by GovOPlaN deployment" if required else ":-" return {key: f"${{{key}{suffix}}}" for key in keys}