573 lines
19 KiB
Python
573 lines
19 KiB
Python
"""Secret handling and deterministic Compose bundle rendering."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
from dataclasses import dataclass
|
|
from hashlib import sha256
|
|
import hmac
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import stat
|
|
from typing import Mapping
|
|
from urllib.parse import quote, urlsplit
|
|
|
|
from .model import ENV_NAME_PATTERN, InstallationSpec
|
|
|
|
|
|
SPEC_FILENAME = "installation.json"
|
|
ENV_FILENAME = "secrets.env"
|
|
COMPOSE_FILENAME = "compose.json"
|
|
PLAN_FILENAME = "plan.json"
|
|
RECEIPT_FILENAME = "receipt.json"
|
|
LOCK_FILENAME = ".deployment.lock"
|
|
RUNTIME_ENV_KEYS = (
|
|
"APP_ENV",
|
|
"GOVOPLAN_INSTALL_PROFILE",
|
|
"MASTER_KEY_B64",
|
|
"DATABASE_URL",
|
|
"GOVOPLAN_DATABASE_URL_PGTOOLS",
|
|
"ENABLED_MODULES",
|
|
"CELERY_ENABLED",
|
|
"CELERY_QUEUES",
|
|
"REDIS_URL",
|
|
"CORS_ORIGINS",
|
|
"GOVOPLAN_TRUSTED_HOSTS",
|
|
"FORWARDED_ALLOW_IPS",
|
|
"AUTH_COOKIE_SECURE",
|
|
"AUTH_COOKIE_SAMESITE",
|
|
"GOVOPLAN_HTTP_HSTS_SECONDS",
|
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
|
|
"GOVOPLAN_MIGRATION_TRACK",
|
|
"DEV_AUTO_MIGRATE_ENABLED",
|
|
"DEV_BOOTSTRAP_ENABLED",
|
|
"GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE",
|
|
"GOVOPLAN_DEPLOYMENT_SPEC_PATH",
|
|
"FILE_STORAGE_BACKEND",
|
|
"FILE_STORAGE_LOCAL_ROOT",
|
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
|
"FILE_STORAGE_S3_REGION",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
|
"FILE_STORAGE_S3_BUCKET",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BundlePaths:
|
|
root: Path
|
|
spec: Path
|
|
env: Path
|
|
compose: Path
|
|
plan: Path
|
|
receipt: Path
|
|
lock: Path
|
|
|
|
|
|
def bundle_paths(root: Path) -> BundlePaths:
|
|
expanded = root.expanduser()
|
|
if expanded.is_symlink():
|
|
raise ValueError(
|
|
f"installation root must not be a symbolic link: {expanded}"
|
|
)
|
|
resolved = expanded.resolve()
|
|
return BundlePaths(
|
|
root=resolved,
|
|
spec=resolved / SPEC_FILENAME,
|
|
env=resolved / ENV_FILENAME,
|
|
compose=resolved / COMPOSE_FILENAME,
|
|
plan=resolved / PLAN_FILENAME,
|
|
receipt=resolved / RECEIPT_FILENAME,
|
|
lock=resolved / LOCK_FILENAME,
|
|
)
|
|
|
|
|
|
def ensure_private_directory(path: Path) -> None:
|
|
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
if path.is_symlink() or not path.is_dir():
|
|
raise ValueError(f"installation root must be a real directory: {path}")
|
|
current = stat.S_IMODE(path.stat().st_mode)
|
|
if current & 0o077:
|
|
path.chmod(0o700)
|
|
|
|
|
|
def initial_secrets(
|
|
spec: InstallationSpec,
|
|
*,
|
|
supplied: Mapping[str, str] | None = None,
|
|
) -> dict[str, str]:
|
|
values = dict(supplied or {})
|
|
values.setdefault(
|
|
"MASTER_KEY_B64",
|
|
base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"),
|
|
)
|
|
values.setdefault("POSTGRES_DB", "govoplan")
|
|
values.setdefault("POSTGRES_USER", "govoplan")
|
|
values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36))
|
|
values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36))
|
|
return reconcile_runtime_environment(spec, values)
|
|
|
|
|
|
def reconcile_runtime_environment(
|
|
spec: InstallationSpec,
|
|
current: Mapping[str, str],
|
|
) -> dict[str, str]:
|
|
values = dict(current)
|
|
postgres = spec.components.postgres
|
|
if postgres.mode == "managed":
|
|
database = values.setdefault("POSTGRES_DB", "govoplan")
|
|
username = values.setdefault("POSTGRES_USER", "govoplan")
|
|
password = values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36))
|
|
encoded_user = quote(username, safe="")
|
|
encoded_password = quote(password, safe="")
|
|
encoded_database = quote(database, safe="")
|
|
values["DATABASE_URL"] = (
|
|
f"postgresql+psycopg://{encoded_user}:{encoded_password}"
|
|
f"@postgres:5432/{encoded_database}"
|
|
)
|
|
values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = (
|
|
f"postgresql://{encoded_user}:{encoded_password}"
|
|
f"@postgres:5432/{encoded_database}"
|
|
)
|
|
elif not values.get(postgres.url_env):
|
|
raise ValueError(
|
|
f"external PostgreSQL requires {postgres.url_env} in {ENV_FILENAME}"
|
|
)
|
|
else:
|
|
_validate_service_url(
|
|
values[postgres.url_env],
|
|
schemes={"postgresql", "postgresql+psycopg"},
|
|
label="external PostgreSQL URL",
|
|
)
|
|
|
|
redis = spec.components.redis
|
|
if redis.mode == "managed":
|
|
password = values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36))
|
|
values["REDIS_URL"] = f"redis://:{quote(password, safe='')}@redis:6379/0"
|
|
elif redis.mode == "external":
|
|
if not values.get(redis.url_env):
|
|
raise ValueError(
|
|
f"external Redis requires {redis.url_env} in {ENV_FILENAME}"
|
|
)
|
|
_validate_service_url(
|
|
values[redis.url_env],
|
|
schemes={"redis", "rediss"},
|
|
label="external Redis URL",
|
|
)
|
|
else:
|
|
values["REDIS_URL"] = ""
|
|
|
|
public = urlsplit(spec.public_url)
|
|
values.update(
|
|
{
|
|
"APP_ENV": "production" if spec.profile == "self-hosted" else "staging",
|
|
"GOVOPLAN_INSTALL_PROFILE": spec.profile,
|
|
"ENABLED_MODULES": ",".join(spec.enabled_modules),
|
|
"CELERY_ENABLED": "true" if redis.mode != "disabled" else "false",
|
|
"CELERY_QUEUES": (
|
|
"send_email,append_sent,notifications,calendar,dataflow,events,default"
|
|
),
|
|
"CORS_ORIGINS": spec.public_url,
|
|
"GOVOPLAN_TRUSTED_HOSTS": public.hostname or "",
|
|
"FORWARDED_ALLOW_IPS": spec.network_subnet,
|
|
"AUTH_COOKIE_SECURE": "true" if public.scheme == "https" else "false",
|
|
"AUTH_COOKIE_SAMESITE": "lax",
|
|
"GOVOPLAN_HTTP_HSTS_SECONDS": (
|
|
"31536000" if public.scheme == "https" else "0"
|
|
),
|
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false",
|
|
"GOVOPLAN_MIGRATION_TRACK": "release",
|
|
"DEV_AUTO_MIGRATE_ENABLED": "false",
|
|
"DEV_BOOTSTRAP_ENABLED": "false",
|
|
"GOVOPLAN_DEPLOYMENT_SPEC_PATH": "/etc/govoplan/deployment/installation.json",
|
|
}
|
|
)
|
|
if redis.mode == "disabled":
|
|
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true"
|
|
else:
|
|
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false"
|
|
|
|
storage = spec.components.storage
|
|
values["FILE_STORAGE_BACKEND"] = storage.mode
|
|
if storage.mode == "local":
|
|
values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files"
|
|
else:
|
|
required = (
|
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
|
"FILE_STORAGE_S3_REGION",
|
|
"FILE_STORAGE_S3_ACCESS_KEY_ID",
|
|
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
|
|
"FILE_STORAGE_S3_BUCKET",
|
|
)
|
|
missing = [name for name in required if not values.get(name)]
|
|
if missing:
|
|
raise ValueError(
|
|
"S3 storage requires values in secrets.env: " + ", ".join(missing)
|
|
)
|
|
endpoint = _validate_service_url(
|
|
values["FILE_STORAGE_S3_ENDPOINT_URL"],
|
|
schemes={"http", "https"},
|
|
label="S3 endpoint URL",
|
|
)
|
|
if spec.profile == "self-hosted" and endpoint.scheme != "https":
|
|
raise ValueError("self-hosted S3 endpoint URL must use HTTPS")
|
|
return dict(sorted(values.items()))
|
|
|
|
|
|
def render_compose(spec: InstallationSpec) -> dict[str, object]:
|
|
runtime_env = {
|
|
"environment": _environment_references(RUNTIME_ENV_KEYS),
|
|
}
|
|
deployment_mount = (
|
|
f"./{SPEC_FILENAME}:/etc/govoplan/deployment/installation.json:ro"
|
|
)
|
|
data_mounts = [deployment_mount]
|
|
if spec.components.storage.mode == "local":
|
|
data_mounts.append("files-data:/var/lib/govoplan/files")
|
|
|
|
dependency_conditions: dict[str, dict[str, str]] = {}
|
|
services: dict[str, object] = {}
|
|
if spec.components.postgres.mode == "managed":
|
|
services["postgres"] = {
|
|
"image": spec.components.postgres.image,
|
|
"restart": "unless-stopped",
|
|
"environment": _environment_references(
|
|
("POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"),
|
|
required=True,
|
|
),
|
|
"healthcheck": {
|
|
"test": [
|
|
"CMD-SHELL",
|
|
'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"',
|
|
],
|
|
"interval": "5s",
|
|
"timeout": "3s",
|
|
"retries": 30,
|
|
},
|
|
"volumes": ["postgres-data:/var/lib/postgresql/data"],
|
|
"networks": ["internal"],
|
|
}
|
|
dependency_conditions["postgres"] = {"condition": "service_healthy"}
|
|
|
|
if spec.components.redis.mode == "managed":
|
|
services["redis"] = {
|
|
"image": spec.components.redis.image,
|
|
"restart": "unless-stopped",
|
|
"environment": _environment_references(
|
|
("REDIS_PASSWORD",),
|
|
required=True,
|
|
),
|
|
"command": [
|
|
"sh",
|
|
"-ec",
|
|
'exec redis-server --appendonly yes --requirepass "$$REDIS_PASSWORD"',
|
|
],
|
|
"healthcheck": {
|
|
"test": [
|
|
"CMD-SHELL",
|
|
'redis-cli -a "$${REDIS_PASSWORD}" --no-auth-warning ping',
|
|
],
|
|
"interval": "5s",
|
|
"timeout": "3s",
|
|
"retries": 30,
|
|
},
|
|
"volumes": ["redis-data:/data"],
|
|
"networks": ["internal"],
|
|
}
|
|
dependency_conditions["redis"] = {"condition": "service_healthy"}
|
|
|
|
if spec.components.mail.mode == "test-mail":
|
|
services["test-mail"] = {
|
|
"image": spec.components.mail.image,
|
|
"restart": "unless-stopped",
|
|
"environment": {
|
|
"GREENMAIL_OPTS": (
|
|
"-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap "
|
|
"-Dgreenmail.hostname=0.0.0.0"
|
|
)
|
|
},
|
|
"networks": ["internal"],
|
|
}
|
|
|
|
common_runtime: dict[str, object] = {
|
|
**runtime_env,
|
|
"restart": "unless-stopped",
|
|
"volumes": data_mounts,
|
|
"networks": ["internal"],
|
|
}
|
|
if dependency_conditions:
|
|
common_runtime["depends_on"] = dependency_conditions
|
|
|
|
services["migrate"] = {
|
|
**runtime_env,
|
|
"image": spec.release.api_image,
|
|
"command": ["python", "-m", "govoplan_core.commands.init_db"],
|
|
"restart": "no",
|
|
"volumes": data_mounts,
|
|
"networks": ["internal"],
|
|
**({"depends_on": dependency_conditions} if dependency_conditions else {}),
|
|
}
|
|
services["api"] = {
|
|
**common_runtime,
|
|
"image": spec.release.api_image,
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"uvicorn",
|
|
"govoplan_core.server.app:app",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
"8000",
|
|
"--proxy-headers",
|
|
],
|
|
"healthcheck": {
|
|
"test": [
|
|
"CMD",
|
|
"python",
|
|
"-c",
|
|
(
|
|
"import urllib.request;"
|
|
"urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)"
|
|
),
|
|
],
|
|
"interval": "10s",
|
|
"timeout": "5s",
|
|
"retries": 30,
|
|
"start_period": "20s",
|
|
},
|
|
}
|
|
services["web"] = {
|
|
"image": spec.release.web_image,
|
|
"restart": "unless-stopped",
|
|
"environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"},
|
|
"depends_on": {"api": {"condition": "service_healthy"}},
|
|
"ports": [_published_port(spec.listen.address, spec.listen.port, 8080)],
|
|
"networks": ["internal"],
|
|
}
|
|
if spec.components.redis.mode != "disabled":
|
|
services["worker"] = {
|
|
**common_runtime,
|
|
"image": spec.release.api_image,
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"worker",
|
|
"--queues",
|
|
(
|
|
"send_email,append_sent,notifications,calendar,"
|
|
"dataflow,events,default"
|
|
),
|
|
"--loglevel",
|
|
"INFO",
|
|
],
|
|
}
|
|
services["scheduler"] = {
|
|
**common_runtime,
|
|
"image": spec.release.api_image,
|
|
"command": [
|
|
"python",
|
|
"-m",
|
|
"celery",
|
|
"-A",
|
|
"govoplan_core.celery_app:celery",
|
|
"beat",
|
|
"--loglevel",
|
|
"INFO",
|
|
],
|
|
}
|
|
volumes: dict[str, object] = {}
|
|
if spec.components.postgres.mode == "managed":
|
|
volumes["postgres-data"] = {}
|
|
if spec.components.redis.mode == "managed":
|
|
volumes["redis-data"] = {}
|
|
if spec.components.storage.mode == "local":
|
|
volumes["files-data"] = {}
|
|
|
|
return {
|
|
"name": spec.installation_id,
|
|
"services": services,
|
|
"volumes": volumes,
|
|
"networks": {
|
|
"internal": {
|
|
"driver": "bridge",
|
|
"ipam": {"config": [{"subnet": spec.network_subnet}]},
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def service_names(spec: InstallationSpec) -> tuple[str, ...]:
|
|
return tuple(render_compose(spec)["services"].keys())
|
|
|
|
|
|
def canonical_json(value: object) -> bytes:
|
|
return (
|
|
json.dumps(value, indent=2, sort_keys=True, separators=(",", ": "))
|
|
+ "\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def digest_json(value: object) -> str:
|
|
return sha256(canonical_json(value)).hexdigest()
|
|
|
|
|
|
def environment_fingerprint(values: Mapping[str, str]) -> str:
|
|
key = values.get("MASTER_KEY_B64", "").encode("utf-8")
|
|
if not key:
|
|
return ""
|
|
payload = canonical_json(dict(sorted(values.items())))
|
|
return hmac.new(key, payload, sha256).hexdigest()
|
|
|
|
|
|
def read_env(path: Path) -> dict[str, str]:
|
|
if not path.exists():
|
|
return {}
|
|
values: dict[str, str] = {}
|
|
for line_number, raw_line in enumerate(
|
|
path.read_text(encoding="utf-8").splitlines(), start=1
|
|
):
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
key, separator, raw_value = line.partition("=")
|
|
if not separator or not ENV_NAME_PATTERN.fullmatch(key):
|
|
raise ValueError(f"invalid environment line {line_number} in {path}")
|
|
if key in values:
|
|
raise ValueError(
|
|
f"duplicate environment key {key!r} on line {line_number}"
|
|
)
|
|
value = raw_value
|
|
if value.startswith('"'):
|
|
try:
|
|
decoded = json.loads(value)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(
|
|
f"invalid quoted environment value on line {line_number}"
|
|
) from exc
|
|
if not isinstance(decoded, str):
|
|
raise ValueError(
|
|
f"environment value on line {line_number} must be a string"
|
|
)
|
|
value = decoded
|
|
elif value.startswith("'"):
|
|
value = _single_quoted_env_value(value, line_number=line_number)
|
|
values[key] = value
|
|
return values
|
|
|
|
|
|
def write_env(path: Path, values: Mapping[str, str]) -> None:
|
|
invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key))
|
|
if invalid:
|
|
raise ValueError(
|
|
"invalid environment variable names: " + ", ".join(invalid)
|
|
)
|
|
lines = [
|
|
"# Generated by govoplan-deploy. Keep this file private.",
|
|
*[f"{key}={_env_value(value)}" for key, value in sorted(values.items())],
|
|
"",
|
|
]
|
|
atomic_write(path, "\n".join(lines).encode("utf-8"), mode=0o600)
|
|
|
|
|
|
def atomic_write(path: Path, payload: bytes, *, mode: int) -> None:
|
|
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
temporary = path.with_name(
|
|
f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
|
|
)
|
|
descriptor = os.open(
|
|
temporary,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
mode,
|
|
)
|
|
try:
|
|
with os.fdopen(descriptor, "wb", closefd=True) as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, path)
|
|
path.chmod(mode)
|
|
directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
finally:
|
|
if temporary.exists():
|
|
temporary.unlink()
|
|
|
|
|
|
def _env_value(value: str) -> str:
|
|
if "\x00" in value or "\n" in value or "\r" in value:
|
|
raise ValueError("environment values must not contain NUL or line breaks")
|
|
if value and all(
|
|
character.isalnum() or character in "_./:@%+,-"
|
|
for character in value
|
|
):
|
|
return value
|
|
escaped = value.replace("\\", "\\\\").replace("'", "\\'")
|
|
return f"'{escaped}'"
|
|
|
|
|
|
def _validate_service_url(
|
|
value: str,
|
|
*,
|
|
schemes: set[str],
|
|
label: str,
|
|
):
|
|
parsed = urlsplit(value)
|
|
try:
|
|
parsed.port
|
|
except ValueError as exc:
|
|
raise ValueError(f"{label} contains an invalid port") from exc
|
|
if parsed.scheme not in schemes or not parsed.hostname:
|
|
raise ValueError(
|
|
f"{label} must use one of {', '.join(sorted(schemes))} and include a host"
|
|
)
|
|
if parsed.fragment:
|
|
raise ValueError(f"{label} must not contain a fragment")
|
|
return parsed
|
|
|
|
|
|
def _single_quoted_env_value(value: str, *, line_number: int) -> str:
|
|
if len(value) < 2 or not value.endswith("'"):
|
|
raise ValueError(
|
|
f"unterminated single-quoted environment value on line {line_number}"
|
|
)
|
|
body = value[1:-1]
|
|
output: list[str] = []
|
|
index = 0
|
|
while index < len(body):
|
|
character = body[index]
|
|
if character != "\\":
|
|
output.append(character)
|
|
index += 1
|
|
continue
|
|
index += 1
|
|
if index >= len(body) or body[index] not in {"\\", "'"}:
|
|
raise ValueError(
|
|
f"invalid single-quoted escape on line {line_number}"
|
|
)
|
|
output.append(body[index])
|
|
index += 1
|
|
return "".join(output)
|
|
|
|
|
|
def _published_port(address: str, host_port: int, container_port: int) -> str:
|
|
host = f"[{address}]" if ":" in address else address
|
|
return f"{host}:{host_port}:{container_port}"
|
|
|
|
|
|
def _environment_references(
|
|
keys: tuple[str, ...],
|
|
*,
|
|
required: bool = False,
|
|
) -> dict[str, str]:
|
|
suffix = ":?required by GovOPlaN deployment" if required else ":-"
|
|
return {key: f"${{{key}{suffix}}}" for key in keys}
|