"""Secret handling and deterministic Compose bundle rendering.""" from __future__ import annotations import base64 from dataclasses import dataclass from hashlib import sha256 import hmac import json import os from pathlib import Path import secrets import stat from typing import Mapping from urllib.parse import quote, urlsplit from .model import ENV_NAME_PATTERN, InstallationSpec SPEC_FILENAME = "installation.json" ENV_FILENAME = "secrets.env" COMPOSE_FILENAME = "compose.json" GARAGE_CONFIG_FILENAME = "garage.toml" LOAD_BALANCER_CONFIG_FILENAME = "load-balancer.cfg" PLAN_FILENAME = "plan.json" RECEIPT_FILENAME = "receipt.json" LOCK_FILENAME = ".deployment.lock" RUNTIME_ENV_KEYS = ( "APP_ENV", "GOVOPLAN_INSTALL_PROFILE", "MASTER_KEY_B64", "DATABASE_URL", "GOVOPLAN_DATABASE_URL_PGTOOLS", "ENABLED_MODULES", "CELERY_ENABLED", "CELERY_QUEUES", "REDIS_URL", "CORS_ORIGINS", "GOVOPLAN_TRUSTED_HOSTS", "FORWARDED_ALLOW_IPS", "AUTH_COOKIE_SECURE", "AUTH_COOKIE_SAMESITE", "GOVOPLAN_HTTP_HSTS_SECONDS", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", "GOVOPLAN_MIGRATION_TRACK", "DEV_AUTO_MIGRATE_ENABLED", "DEV_BOOTSTRAP_ENABLED", "GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE", "GOVOPLAN_DEPLOYMENT_SPEC_PATH", "FILE_STORAGE_BACKEND", "FILE_STORAGE_LOCAL_ROOT", "FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET", "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", ) @dataclass(frozen=True, slots=True) class BundlePaths: root: Path spec: Path env: Path compose: Path garage_config: Path load_balancer_config: Path plan: Path receipt: Path lock: Path def bundle_paths(root: Path) -> BundlePaths: expanded = root.expanduser() if expanded.is_symlink(): raise ValueError(f"installation root must not be a symbolic link: {expanded}") resolved = expanded.resolve() return BundlePaths( root=resolved, spec=resolved / SPEC_FILENAME, env=resolved / ENV_FILENAME, compose=resolved / COMPOSE_FILENAME, garage_config=resolved / GARAGE_CONFIG_FILENAME, load_balancer_config=resolved / LOAD_BALANCER_CONFIG_FILENAME, plan=resolved / PLAN_FILENAME, receipt=resolved / RECEIPT_FILENAME, lock=resolved / LOCK_FILENAME, ) def ensure_private_directory(path: Path) -> None: path.mkdir(mode=0o700, parents=True, exist_ok=True) if path.is_symlink() or not path.is_dir(): raise ValueError(f"installation root must be a real directory: {path}") current = stat.S_IMODE(path.stat().st_mode) if current & 0o077: path.chmod(0o700) def initial_secrets( spec: InstallationSpec, *, supplied: Mapping[str, str] | None = None, ) -> dict[str, str]: values = dict(supplied or {}) values.setdefault( "MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"), ) values.setdefault("POSTGRES_DB", "govoplan") values.setdefault("POSTGRES_USER", "govoplan") values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) return reconcile_runtime_environment(spec, values) def reconcile_runtime_environment( spec: InstallationSpec, current: Mapping[str, str], ) -> dict[str, str]: values = dict(current) postgres = spec.components.postgres if postgres.mode == "managed": database = values.setdefault("POSTGRES_DB", "govoplan") username = values.setdefault("POSTGRES_USER", "govoplan") password = values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) encoded_user = quote(username, safe="") encoded_password = quote(password, safe="") encoded_database = quote(database, safe="") values["DATABASE_URL"] = ( f"postgresql+psycopg://{encoded_user}:{encoded_password}" f"@postgres:5432/{encoded_database}" ) values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = ( f"postgresql://{encoded_user}:{encoded_password}" f"@postgres:5432/{encoded_database}" ) elif not values.get(postgres.url_env): raise ValueError( f"external PostgreSQL requires {postgres.url_env} in {ENV_FILENAME}" ) else: _validate_service_url( values[postgres.url_env], schemes={"postgresql", "postgresql+psycopg"}, label="external PostgreSQL URL", ) redis = spec.components.redis if redis.mode == "managed": password = values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) values["REDIS_URL"] = f"redis://:{quote(password, safe='')}@redis:6379/0" elif redis.mode == "external": if not values.get(redis.url_env): raise ValueError( f"external Redis requires {redis.url_env} in {ENV_FILENAME}" ) _validate_service_url( values[redis.url_env], schemes={"redis", "rediss"}, label="external Redis URL", ) else: values["REDIS_URL"] = "" public = urlsplit(spec.public_url) values.update( { "APP_ENV": "production" if spec.profile == "self-hosted" else "staging", "GOVOPLAN_INSTALL_PROFILE": spec.profile, "ENABLED_MODULES": ",".join(spec.enabled_modules), "CELERY_ENABLED": "true" if redis.mode != "disabled" else "false", "CELERY_QUEUES": ( "send_email,append_sent,notifications,calendar,dataflow,workflow,events,default" ), "CORS_ORIGINS": spec.public_url, "GOVOPLAN_TRUSTED_HOSTS": public.hostname or "", "FORWARDED_ALLOW_IPS": spec.network_subnet, "AUTH_COOKIE_SECURE": "true" if public.scheme == "https" else "false", "AUTH_COOKIE_SAMESITE": "lax", "GOVOPLAN_HTTP_HSTS_SECONDS": ( "31536000" if public.scheme == "https" else "0" ), "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false", "GOVOPLAN_MIGRATION_TRACK": "release", "DEV_AUTO_MIGRATE_ENABLED": "false", "DEV_BOOTSTRAP_ENABLED": "false", "GOVOPLAN_DEPLOYMENT_SPEC_PATH": "/etc/govoplan/deployment/installation.json", } ) if redis.mode == "disabled": values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true" else: values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false" storage = spec.components.storage if storage.mode == "local": values["FILE_STORAGE_BACKEND"] = "local" values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false" values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files" elif storage.mode == "garage": access_key = values.setdefault( "GARAGE_DEFAULT_ACCESS_KEY", f"GK{secrets.token_hex(16)}", ) secret_key = values.setdefault( "GARAGE_DEFAULT_SECRET_KEY", secrets.token_hex(32), ) bucket = values.setdefault("GARAGE_DEFAULT_BUCKET", "govoplan-files") values.setdefault("GARAGE_RPC_SECRET", secrets.token_hex(32)) values.setdefault("GARAGE_ADMIN_TOKEN", secrets.token_urlsafe(48)) values.setdefault("GARAGE_METRICS_TOKEN", secrets.token_urlsafe(48)) values.update( { "FILE_STORAGE_BACKEND": "s3", "FILE_STORAGE_S3_ENDPOINT_URL": "http://garage:3900", "FILE_STORAGE_S3_REGION": "garage", "FILE_STORAGE_S3_ACCESS_KEY_ID": access_key, "FILE_STORAGE_S3_SECRET_ACCESS_KEY": secret_key, "FILE_STORAGE_S3_BUCKET": bucket, "FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "true", } ) else: values["FILE_STORAGE_BACKEND"] = "s3" values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false" required = ( "FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET", ) missing = [name for name in required if not values.get(name)] if missing: raise ValueError( "S3 storage requires values in secrets.env: " + ", ".join(missing) ) endpoint = _validate_service_url( values["FILE_STORAGE_S3_ENDPOINT_URL"], schemes={"http", "https"}, label="S3 endpoint URL", ) if spec.profile == "self-hosted" and endpoint.scheme != "https": raise ValueError("self-hosted S3 endpoint URL must use HTTPS") return dict(sorted(values.items())) def render_compose(spec: InstallationSpec) -> dict[str, object]: runtime_env = { "environment": _environment_references(RUNTIME_ENV_KEYS), } deployment_mount = ( f"./{SPEC_FILENAME}:/etc/govoplan/deployment/installation.json:ro" ) data_mounts = [deployment_mount] if spec.components.storage.mode == "local": data_mounts.append("files-data:/var/lib/govoplan/files") dependency_conditions: dict[str, dict[str, str]] = {} services: dict[str, object] = {} if spec.components.postgres.mode == "managed": services["postgres"] = { "image": spec.components.postgres.image, "restart": "unless-stopped", "environment": _environment_references( ("POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"), required=True, ), "healthcheck": { "test": [ "CMD-SHELL", 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"', ], "interval": "5s", "timeout": "3s", "retries": 30, }, "volumes": ["postgres-data:/var/lib/postgresql/data"], "networks": ["internal"], } dependency_conditions["postgres"] = {"condition": "service_healthy"} if spec.components.redis.mode == "managed": services["redis"] = { "image": spec.components.redis.image, "restart": "unless-stopped", "environment": _environment_references( ("REDIS_PASSWORD",), required=True, ), "command": [ "sh", "-ec", 'exec redis-server --appendonly yes --requirepass "$$REDIS_PASSWORD"', ], "healthcheck": { "test": [ "CMD-SHELL", 'redis-cli -a "$${REDIS_PASSWORD}" --no-auth-warning ping', ], "interval": "5s", "timeout": "3s", "retries": 30, }, "volumes": ["redis-data:/data"], "networks": ["internal"], } dependency_conditions["redis"] = {"condition": "service_healthy"} if spec.components.storage.mode == "garage": garage_environment = _environment_references( ( "GARAGE_DEFAULT_ACCESS_KEY", "GARAGE_DEFAULT_SECRET_KEY", "GARAGE_DEFAULT_BUCKET", "GARAGE_RPC_SECRET", "GARAGE_ADMIN_TOKEN", "GARAGE_METRICS_TOKEN", ), required=True, ) services["garage"] = { "image": spec.components.storage.image, "restart": "unless-stopped", "command": [ "/garage", "server", "--single-node", "--default-bucket", ], "environment": garage_environment, "healthcheck": { "test": ["CMD", "/garage", "status"], "interval": "10s", "timeout": "5s", "retries": 30, "start_period": "15s", }, "labels": { "org.govoplan.configuration-sha256": sha256( render_garage_config().encode("utf-8") ).hexdigest() }, "security_opt": ["no-new-privileges:true"], "volumes": [ f"./{GARAGE_CONFIG_FILENAME}:/etc/garage.toml:ro", "garage-meta:/var/lib/garage/meta", "garage-data:/var/lib/garage/data", ], "networks": ["internal"], } dependency_conditions["garage"] = {"condition": "service_healthy"} if spec.components.mail.mode == "test-mail": services["test-mail"] = { "image": spec.components.mail.image, "restart": "unless-stopped", "environment": { "GREENMAIL_OPTS": ( "-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap " "-Dgreenmail.hostname=0.0.0.0" ) }, "networks": ["internal"], } common_runtime: dict[str, object] = { **runtime_env, "restart": "unless-stopped", "volumes": data_mounts, "networks": ["internal"], } if dependency_conditions: common_runtime["depends_on"] = dependency_conditions services["migrate"] = { **runtime_env, "image": spec.release.api_image, "command": ["python", "-m", "govoplan_core.commands.init_db"], "restart": "no", "volumes": data_mounts, "networks": ["internal"], **({"depends_on": dependency_conditions} if dependency_conditions else {}), } services["api"] = { **common_runtime, "image": spec.release.api_image, "scale": spec.replicas.api, "command": [ "python", "-m", "uvicorn", "govoplan_core.server.app:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", ], "healthcheck": { "test": [ "CMD", "python", "-c", ( "import urllib.request;" "urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)" ), ], "interval": "10s", "timeout": "5s", "retries": 30, "start_period": "20s", }, } services["web"] = { "image": spec.release.web_image, "restart": "unless-stopped", "scale": spec.replicas.web, "environment": {"GOVOPLAN_API_UPSTREAM": "http://load-balancer:8000"}, "networks": ["internal"], } services["load-balancer"] = { "image": spec.components.load_balancer.image, "restart": "unless-stopped", "healthcheck": { "test": [ "CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg", ], "interval": "10s", "timeout": "5s", "retries": 10, }, "labels": { "org.govoplan.configuration-sha256": sha256( render_load_balancer_config(spec).encode("utf-8") ).hexdigest() }, "ports": [_published_port(spec.listen.address, spec.listen.port, 8080)], "read_only": True, "security_opt": ["no-new-privileges:true"], "volumes": [ (f"./{LOAD_BALANCER_CONFIG_FILENAME}:/usr/local/etc/haproxy/haproxy.cfg:ro") ], "networks": ["internal"], } if spec.components.redis.mode != "disabled": services["worker"] = { **common_runtime, "image": spec.release.api_image, "scale": spec.replicas.worker, "command": [ "python", "-m", "celery", "-A", "govoplan_core.celery_app:celery", "worker", "--queues", ( "send_email,append_sent,notifications,calendar," "dataflow,events,default" ), "--loglevel", "INFO", ], } services["scheduler"] = { **common_runtime, "image": spec.release.api_image, "command": [ "python", "-m", "celery", "-A", "govoplan_core.celery_app:celery", "beat", "--loglevel", "INFO", ], } volumes: dict[str, object] = {} if spec.components.postgres.mode == "managed": volumes["postgres-data"] = {} if spec.components.redis.mode == "managed": volumes["redis-data"] = {} if spec.components.storage.mode == "local": volumes["files-data"] = {} if spec.components.storage.mode == "garage": volumes["garage-meta"] = {} volumes["garage-data"] = {} return { "name": spec.installation_id, "services": services, "volumes": volumes, "networks": { "internal": { "driver": "bridge", "ipam": {"config": [{"subnet": spec.network_subnet}]}, } }, } def render_garage_config() -> str: return """metadata_dir = "/var/lib/garage/meta" data_dir = "/var/lib/garage/data" db_engine = "sqlite" replication_factor = 1 rpc_bind_addr = "[::]:3901" rpc_public_addr = "127.0.0.1:3901" [s3_api] s3_region = "garage" api_bind_addr = "[::]:3900" root_domain = ".s3.garage.localhost" [admin] api_bind_addr = "[::]:3903" """ def render_load_balancer_config(spec: InstallationSpec) -> str: return f"""global log stdout format raw local0 maxconn 4096 defaults log global mode http option httplog option redispatch timeout connect 5s timeout client 60s timeout server 60s resolvers docker nameserver dns 127.0.0.11:53 resolve_retries 3 timeout resolve 1s timeout retry 1s hold other 10s hold refused 10s hold nx 10s hold timeout 10s hold valid 10s hold obsolete 10s frontend public_web bind :8080 default_backend web_replicas backend web_replicas balance roundrobin option httpchk GET /health http-check expect status 200 server-template web- {spec.replicas.web} web:8080 check resolvers docker init-addr libc,none frontend internal_api bind :8000 default_backend api_replicas backend api_replicas balance leastconn option httpchk GET /health http-check expect status 200 server-template api- {spec.replicas.api} api:8000 check resolvers docker init-addr libc,none """ def service_names(spec: InstallationSpec) -> tuple[str, ...]: return tuple(render_compose(spec)["services"].keys()) def canonical_json(value: object) -> bytes: return ( json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + "\n" ).encode("utf-8") def digest_json(value: object) -> str: return sha256(canonical_json(value)).hexdigest() def environment_fingerprint(values: Mapping[str, str]) -> str: key = values.get("MASTER_KEY_B64", "").encode("utf-8") if not key: return "" payload = canonical_json(dict(sorted(values.items()))) return hmac.new(key, payload, sha256).hexdigest() def read_env(path: Path) -> dict[str, str]: if not path.exists(): return {} values: dict[str, str] = {} for line_number, raw_line in enumerate( path.read_text(encoding="utf-8").splitlines(), start=1 ): line = raw_line.strip() if not line or line.startswith("#"): continue key, separator, raw_value = line.partition("=") if not separator or not ENV_NAME_PATTERN.fullmatch(key): raise ValueError(f"invalid environment line {line_number} in {path}") if key in values: raise ValueError(f"duplicate environment key {key!r} on line {line_number}") value = raw_value if value.startswith('"'): try: decoded = json.loads(value) except json.JSONDecodeError as exc: raise ValueError( f"invalid quoted environment value on line {line_number}" ) from exc if not isinstance(decoded, str): raise ValueError( f"environment value on line {line_number} must be a string" ) value = decoded elif value.startswith("'"): value = _single_quoted_env_value(value, line_number=line_number) values[key] = value return values def write_env(path: Path, values: Mapping[str, str]) -> None: invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key)) if invalid: raise ValueError("invalid environment variable names: " + ", ".join(invalid)) lines = [ "# Generated by govoplan-deploy. Keep this file private.", *[f"{key}={_env_value(value)}" for key, value in sorted(values.items())], "", ] atomic_write(path, "\n".join(lines).encode("utf-8"), mode=0o600) def atomic_write(path: Path, payload: bytes, *, mode: int) -> None: path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp") descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode, ) try: with os.fdopen(descriptor, "wb", closefd=True) as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) path.chmod(mode) directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: if temporary.exists(): temporary.unlink() def _env_value(value: str) -> str: if "\x00" in value or "\n" in value or "\r" in value: raise ValueError("environment values must not contain NUL or line breaks") if value and all( character.isalnum() or character in "_./:@%+,-" for character in value ): return value escaped = value.replace("\\", "\\\\").replace("'", "\\'") return f"'{escaped}'" def _validate_service_url( value: str, *, schemes: set[str], label: str, ): parsed = urlsplit(value) try: parsed.port except ValueError as exc: raise ValueError(f"{label} contains an invalid port") from exc if parsed.scheme not in schemes or not parsed.hostname: raise ValueError( f"{label} must use one of {', '.join(sorted(schemes))} and include a host" ) if parsed.fragment: raise ValueError(f"{label} must not contain a fragment") return parsed def _single_quoted_env_value(value: str, *, line_number: int) -> str: if len(value) < 2 or not value.endswith("'"): raise ValueError( f"unterminated single-quoted environment value on line {line_number}" ) body = value[1:-1] output: list[str] = [] index = 0 while index < len(body): character = body[index] if character != "\\": output.append(character) index += 1 continue index += 1 if index >= len(body) or body[index] not in {"\\", "'"}: raise ValueError(f"invalid single-quoted escape on line {line_number}") output.append(body[index]) index += 1 return "".join(output) def _published_port(address: str, host_port: int, container_port: int) -> str: host = f"[{address}]" if ":" in address else address return f"{host}:{host_port}:{container_port}" def _environment_references( keys: tuple[str, ...], *, required: bool = False, ) -> dict[str, str]: suffix = ":?required by GovOPlaN deployment" if required else ":-" return {key: f"${{{key}{suffix}}}" for key in keys}