GovOPlaN/govoplan
1
0
Fork 0
Code Issues 19 Pull Requests Actions Packages Projects Releases Wiki Activity

Add scalable deployment planning and guided releases
Security Audit / security-audit (push) Failing after 4s
Details
Dependency Audit / dependency-audit (push) Failing after 7s
Details
Deployment Installer / deployment-installer (push) Failing after 4s
Details

Browse Source
This commit is contained in:
zemion
2026-07-31 02:49:03 +02:00
parent 3864ce28b1
commit 908090dd0f
13 changed files with 1511 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/deployment/govoplan_deploy/bundle.py
+188 -22
View File
@@ -20,6 +20,8 @@ from .model import ENV_NAME_PATTERN, InstallationSpec
SPEC_FILENAME = "installation.json"
ENV_FILENAME = "secrets.env"
COMPOSE_FILENAME = "compose.json"
GARAGE_CONFIG_FILENAME = "garage.toml"
LOAD_BALANCER_CONFIG_FILENAME = "load-balancer.cfg"
PLAN_FILENAME = "plan.json"
RECEIPT_FILENAME = "receipt.json"
LOCK_FILENAME = ".deployment.lock"
@@ -52,6 +54,7 @@ RUNTIME_ENV_KEYS = (
"FILE_STORAGE_S3_ACCESS_KEY_ID",
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
)
@@ -61,6 +64,8 @@ class BundlePaths:
spec: Path
env: Path
compose: Path
garage_config: Path
load_balancer_config: Path
plan: Path
receipt: Path
lock: Path
@@ -69,15 +74,15 @@ class BundlePaths:
def bundle_paths(root: Path) -> BundlePaths:
expanded = root.expanduser()
if expanded.is_symlink():
raise ValueError(
f"installation root must not be a symbolic link: {expanded}"
)
raise ValueError(f"installation root must not be a symbolic link: {expanded}")
resolved = expanded.resolve()
return BundlePaths(
root=resolved,
spec=resolved / SPEC_FILENAME,
env=resolved / ENV_FILENAME,
compose=resolved / COMPOSE_FILENAME,
garage_config=resolved / GARAGE_CONFIG_FILENAME,
load_balancer_config=resolved / LOAD_BALANCER_CONFIG_FILENAME,
plan=resolved / PLAN_FILENAME,
receipt=resolved / RECEIPT_FILENAME,
lock=resolved / LOCK_FILENAME,
@@ -190,10 +195,37 @@ def reconcile_runtime_environment(
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false"
storage = spec.components.storage
values["FILE_STORAGE_BACKEND"] = storage.mode
if storage.mode == "local":
values["FILE_STORAGE_BACKEND"] = "local"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files"
elif storage.mode == "garage":
access_key = values.setdefault(
"GARAGE_DEFAULT_ACCESS_KEY",
f"GK{secrets.token_hex(16)}",
)
secret_key = values.setdefault(
"GARAGE_DEFAULT_SECRET_KEY",
secrets.token_hex(32),
)
bucket = values.setdefault("GARAGE_DEFAULT_BUCKET", "govoplan-files")
values.setdefault("GARAGE_RPC_SECRET", secrets.token_hex(32))
values.setdefault("GARAGE_ADMIN_TOKEN", secrets.token_urlsafe(48))
values.setdefault("GARAGE_METRICS_TOKEN", secrets.token_urlsafe(48))
values.update(
{
"FILE_STORAGE_BACKEND": "s3",
"FILE_STORAGE_S3_ENDPOINT_URL": "http://garage:3900",
"FILE_STORAGE_S3_REGION": "garage",
"FILE_STORAGE_S3_ACCESS_KEY_ID": access_key,
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": secret_key,
"FILE_STORAGE_S3_BUCKET": bucket,
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "true",
}
)
else:
values["FILE_STORAGE_BACKEND"] = "s3"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
required = (
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_REGION",
@@ -278,6 +310,50 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
dependency_conditions["redis"] = {"condition": "service_healthy"}
if spec.components.storage.mode == "garage":
garage_environment = _environment_references(
(
"GARAGE_DEFAULT_ACCESS_KEY",
"GARAGE_DEFAULT_SECRET_KEY",
"GARAGE_DEFAULT_BUCKET",
"GARAGE_RPC_SECRET",
"GARAGE_ADMIN_TOKEN",
"GARAGE_METRICS_TOKEN",
),
required=True,
)
services["garage"] = {
"image": spec.components.storage.image,
"restart": "unless-stopped",
"command": [
"/garage",
"server",
"--single-node",
"--default-bucket",
],
"environment": garage_environment,
"healthcheck": {
"test": ["CMD", "/garage", "status"],
"interval": "10s",
"timeout": "5s",
"retries": 30,
"start_period": "15s",
},
"labels": {
"org.govoplan.configuration-sha256": sha256(
render_garage_config().encode("utf-8")
).hexdigest()
},
"security_opt": ["no-new-privileges:true"],
"volumes": [
f"./{GARAGE_CONFIG_FILENAME}:/etc/garage.toml:ro",
"garage-meta:/var/lib/garage/meta",
"garage-data:/var/lib/garage/data",
],
"networks": ["internal"],
}
dependency_conditions["garage"] = {"condition": "service_healthy"}
if spec.components.mail.mode == "test-mail":
services["test-mail"] = {
"image": spec.components.mail.image,
@@ -312,6 +388,7 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
services["api"] = {
**common_runtime,
"image": spec.release.api_image,
"scale": spec.replicas.api,
"command": [
"python",
"-m",
@@ -342,15 +419,43 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
services["web"] = {
"image": spec.release.web_image,
"restart": "unless-stopped",
"environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"},
"depends_on": {"api": {"condition": "service_healthy"}},
"scale": spec.replicas.web,
"environment": {"GOVOPLAN_API_UPSTREAM": "http://load-balancer:8000"},
"networks": ["internal"],
}
services["load-balancer"] = {
"image": spec.components.load_balancer.image,
"restart": "unless-stopped",
"healthcheck": {
"test": [
"CMD",
"haproxy",
"-c",
"-f",
"/usr/local/etc/haproxy/haproxy.cfg",
],
"interval": "10s",
"timeout": "5s",
"retries": 10,
},
"labels": {
"org.govoplan.configuration-sha256": sha256(
render_load_balancer_config(spec).encode("utf-8")
).hexdigest()
},
"ports": [_published_port(spec.listen.address, spec.listen.port, 8080)],
"read_only": True,
"security_opt": ["no-new-privileges:true"],
"volumes": [
(f"./{LOAD_BALANCER_CONFIG_FILENAME}:/usr/local/etc/haproxy/haproxy.cfg:ro")
],
"networks": ["internal"],
}
if spec.components.redis.mode != "disabled":
services["worker"] = {
**common_runtime,
"image": spec.release.api_image,
"scale": spec.replicas.worker,
"command": [
"python",
"-m",
@@ -388,6 +493,9 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
volumes["redis-data"] = {}
if spec.components.storage.mode == "local":
volumes["files-data"] = {}
if spec.components.storage.mode == "garage":
volumes["garage-meta"] = {}
volumes["garage-data"] = {}
return {
"name": spec.installation_id,
@@ -402,14 +510,81 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
def render_garage_config() -> str:
return """metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
[admin]
api_bind_addr = "[::]:3903"
"""
def render_load_balancer_config(spec: InstallationSpec) -> str:
return f"""global
log stdout format raw local0
maxconn 4096
defaults
log global
mode http
option httplog
option redispatch
timeout connect 5s
timeout client 60s
timeout server 60s
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold other 10s
hold refused 10s
hold nx 10s
hold timeout 10s
hold valid 10s
hold obsolete 10s
frontend public_web
bind :8080
default_backend web_replicas
backend web_replicas
balance roundrobin
option httpchk GET /health
http-check expect status 200
server-template web- {spec.replicas.web} web:8080 check resolvers docker init-addr libc,none
frontend internal_api
bind :8000
default_backend api_replicas
backend api_replicas
balance leastconn
option httpchk GET /health
http-check expect status 200
server-template api- {spec.replicas.api} api:8000 check resolvers docker init-addr libc,none
"""
def service_names(spec: InstallationSpec) -> tuple[str, ...]:
return tuple(render_compose(spec)["services"].keys())
def canonical_json(value: object) -> bytes:
return (
json.dumps(value, indent=2, sort_keys=True, separators=(",", ": "))
+ "\n"
json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + "\n"
).encode("utf-8")
@@ -439,9 +614,7 @@ def read_env(path: Path) -> dict[str, str]:
if not separator or not ENV_NAME_PATTERN.fullmatch(key):
raise ValueError(f"invalid environment line {line_number} in {path}")
if key in values:
raise ValueError(
f"duplicate environment key {key!r} on line {line_number}"
)
raise ValueError(f"duplicate environment key {key!r} on line {line_number}")
value = raw_value
if value.startswith('"'):
try:
@@ -464,9 +637,7 @@ def read_env(path: Path) -> dict[str, str]:
def write_env(path: Path, values: Mapping[str, str]) -> None:
invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key))
if invalid:
raise ValueError(
"invalid environment variable names: " + ", ".join(invalid)
)
raise ValueError("invalid environment variable names: " + ", ".join(invalid))
lines = [
"# Generated by govoplan-deploy. Keep this file private.",
*[f"{key}={_env_value(value)}" for key, value in sorted(values.items())],
@@ -477,9 +648,7 @@ def write_env(path: Path, values: Mapping[str, str]) -> None:
def atomic_write(path: Path, payload: bytes, *, mode: int) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
temporary = path.with_name(
f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
)
temporary = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp")
descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
@@ -506,8 +675,7 @@ def _env_value(value: str) -> str:
if "\x00" in value or "\n" in value or "\r" in value:
raise ValueError("environment values must not contain NUL or line breaks")
if value and all(
character.isalnum() or character in "_./:@%+,-"
for character in value
character.isalnum() or character in "_./:@%+,-" for character in value
):
return value
escaped = value.replace("\\", "\\\\").replace("'", "\\'")
@@ -550,9 +718,7 @@ def _single_quoted_env_value(value: str, *, line_number: int) -> str:
continue
index += 1
if index >= len(body) or body[index] not in {"\\", "'"}:
raise ValueError(
f"invalid single-quoted escape on line {line_number}"
)
raise ValueError(f"invalid single-quoted escape on line {line_number}")
output.append(body[index])
index += 1
return "".join(output)
tools/deployment/govoplan_deploy/cli.py
+190 -32
View File
@@ -30,13 +30,18 @@ from .bundle import (
read_env,
reconcile_runtime_environment,
render_compose,
render_garage_config,
render_load_balancer_config,
service_names,
write_env,
)
from .model import (
ComponentConfig,
DEFAULT_GARAGE_IMAGE,
DEFAULT_LOAD_BALANCER_IMAGE,
InstallationSpec,
ListenConfig,
ReplicaConfig,
SpecError,
default_spec,
load_spec,
@@ -117,7 +122,12 @@ def _directory_argument(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--directory",
type=Path,
default=Path.home() / ".local" / "share" / "govoplan" / "installations" / "default",
default=Path.home()
/ ".local"
/ "share"
/ "govoplan"
/ "installations"
/ "default",
help="Private installation state directory.",
)
@@ -140,7 +150,9 @@ def _configuration_arguments(
choices=("managed", "external"),
default=default("managed"),
)
parser.add_argument("--database-url", help="External PostgreSQL URL; stored privately.")
parser.add_argument(
"--database-url", help="External PostgreSQL URL; stored privately."
)
parser.add_argument(
"--redis",
choices=("managed", "external", "disabled"),
@@ -154,17 +166,47 @@ def _configuration_arguments(
)
parser.add_argument(
"--storage",
choices=("local", "s3"),
choices=("local", "garage", "s3"),
default=default("local"),
)
parser.add_argument(
"--garage-image",
default=default(DEFAULT_GARAGE_IMAGE),
help="Garage image used by managed object storage.",
)
parser.add_argument("--s3-endpoint-url", help="S3-compatible endpoint URL.")
parser.add_argument("--s3-region", help="S3 region.")
parser.add_argument("--s3-access-key-id", help="S3 access key id; stored privately.")
parser.add_argument(
"--s3-access-key-id", help="S3 access key id; stored privately."
)
parser.add_argument(
"--s3-secret-access-key",
help="S3 secret access key; stored privately.",
)
parser.add_argument("--s3-bucket", help="S3 bucket for managed files.")
parser.add_argument(
"--load-balancer-image",
default=default(DEFAULT_LOAD_BALANCER_IMAGE),
help="HAProxy image used by the managed local load balancer.",
)
parser.add_argument(
"--api-replicas",
type=int,
default=default(1),
help="API containers on this Compose host (1-64).",
)
parser.add_argument(
"--web-replicas",
type=int,
default=default(1),
help="WebUI containers on this Compose host (1-64).",
)
parser.add_argument(
"--worker-replicas",
type=int,
default=None,
help="Worker containers on this Compose host (1-128, or 0 without Redis).",
)
parser.add_argument(
"--module-set",
choices=("core", "base", "full"),
@@ -216,6 +258,11 @@ def _init(args: argparse.Namespace) -> int:
redis_mode=args.redis,
mail_mode=args.mail,
storage_mode=args.storage,
garage_image=args.garage_image,
load_balancer_image=args.load_balancer_image,
api_replicas=args.api_replicas,
web_replicas=args.web_replicas,
worker_replicas=args.worker_replicas,
module_set=args.module_set,
api_image=args.api_image,
web_image=args.web_image,
@@ -260,9 +307,23 @@ def _configure(args: argparse.Namespace) -> int:
and spec.components.redis.mode == "external"
and not args.redis_url
):
raise ValueError(
"switching Redis to external requires --redis-url"
)
raise ValueError("switching Redis to external requires --redis-url")
if current.components.storage.mode != "s3" and spec.components.storage.mode == "s3":
missing_s3_options = [
flag
for flag, value in (
("--s3-endpoint-url", args.s3_endpoint_url),
("--s3-region", args.s3_region),
("--s3-access-key-id", args.s3_access_key_id),
("--s3-secret-access-key", args.s3_secret_access_key),
("--s3-bucket", args.s3_bucket),
)
if not value
]
if missing_s3_options:
raise ValueError(
"switching to external S3 requires: " + ", ".join(missing_s3_options)
)
secrets = read_env(paths.env)
secrets.update(_supplied_secret_values(args))
secrets = reconcile_runtime_environment(spec, secrets)
@@ -318,6 +379,8 @@ def _apply(args: argparse.Namespace) -> int:
"components.postgres.image.",
"components.redis.image.",
"components.mail.image.",
"components.storage.image.",
"components.load_balancer.image.",
"release.manifest",
"modules.image_composition",
)
@@ -344,15 +407,17 @@ def _apply(args: argparse.Namespace) -> int:
_run([*compose, "pull"], cwd=paths.root)
dependencies = [
name
for name in ("postgres", "redis", "test-mail")
for name in ("postgres", "redis", "garage", "test-mail")
if name in service_names(spec)
]
if dependencies:
_run([*compose, "up", "--detach", *dependencies], cwd=paths.root)
_run([*compose, "run", "--rm", "migrate"], cwd=paths.root)
if _receipt_uses_direct_web_port(paths.receipt):
_run([*compose, "stop", "web"], cwd=paths.root)
runtime_services = [
name
for name in ("api", "web", "worker", "scheduler")
for name in ("api", "web", "load-balancer", "worker", "scheduler")
if name in service_names(spec)
]
_run(
@@ -378,6 +443,11 @@ def _apply(args: argparse.Namespace) -> int:
"web_image": spec.release.web_image,
},
"services": list(service_names(spec)),
"replicas": {
"api": spec.replicas.api,
"web": spec.replicas.web,
"worker": spec.replicas.worker,
},
"listen": {
"address": spec.listen.address,
"port": spec.listen.port,
@@ -470,6 +540,7 @@ def _updated_spec(
api_image=args.api_image or current.release.api_image,
web_image=args.web_image or current.release.web_image,
)
storage_mode = args.storage or current.components.storage.mode
components = ComponentConfig(
postgres=replace(
current.components.postgres,
@@ -485,7 +556,35 @@ def _updated_spec(
),
storage=replace(
current.components.storage,
mode=args.storage or current.components.storage.mode,
mode=storage_mode,
image=(
args.garage_image
or current.components.storage.image
or DEFAULT_GARAGE_IMAGE
)
if storage_mode == "garage"
else "",
),
load_balancer=replace(
current.components.load_balancer,
image=(args.load_balancer_image or current.components.load_balancer.image),
),
)
replicas = ReplicaConfig(
api=(
args.api_replicas if args.api_replicas is not None else current.replicas.api
),
web=(
args.web_replicas if args.web_replicas is not None else current.replicas.web
),
worker=(
args.worker_replicas
if args.worker_replicas is not None
else (
0
if components.redis.mode == "disabled"
else max(current.replicas.worker, 1)
)
),
)
value = replace(
@@ -499,6 +598,7 @@ def _updated_spec(
),
release=release,
components=components,
replicas=replicas,
enabled_modules=modules,
)
return parse_spec(value.to_dict())
@@ -514,9 +614,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]:
s3_values = {
"FILE_STORAGE_S3_ENDPOINT_URL": getattr(args, "s3_endpoint_url", None),
"FILE_STORAGE_S3_REGION": getattr(args, "s3_region", None),
"FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(
args, "s3_access_key_id", None
),
"FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(args, "s3_access_key_id", None),
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": getattr(
args, "s3_secret_access_key", None
),
@@ -528,9 +626,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]:
def _pgtools_url(database_url: str) -> str:
if database_url.startswith("postgresql+psycopg://"):
return "postgresql://" + database_url.removeprefix(
"postgresql+psycopg://"
)
return "postgresql://" + database_url.removeprefix("postgresql+psycopg://")
return database_url
@@ -543,6 +639,16 @@ def _write_bundle(
atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600)
write_env(paths.env, secrets)
atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600)
atomic_write(
paths.load_balancer_config,
render_load_balancer_config(spec).encode("utf-8"),
mode=0o644,
)
atomic_write(
paths.garage_config,
render_garage_config().encode("utf-8"),
mode=0o644,
)
def _write_plan(path: Path, plan: DeploymentPlan) -> None:
@@ -570,9 +676,7 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
if args.profile == "self-hosted" and args.public_url.startswith("http://"):
args.public_url = "https://govoplan.example.org"
args.public_url = _prompt("Public URL", args.public_url)
args.postgres = _prompt_choice(
"PostgreSQL", args.postgres, ("managed", "external")
)
args.postgres = _prompt_choice("PostgreSQL", args.postgres, ("managed", "external"))
if args.postgres == "external" and not args.database_url:
args.database_url = getpass.getpass(
"External PostgreSQL URL (input hidden): "
@@ -582,13 +686,9 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
if args.profile == "self-hosted"
else ("managed", "external", "disabled")
)
args.redis = _prompt_choice(
"Redis", args.redis, redis_choices
)
args.redis = _prompt_choice("Redis", args.redis, redis_choices)
if args.redis == "external" and not args.redis_url:
args.redis_url = getpass.getpass(
"External Redis URL (input hidden): "
).strip()
args.redis_url = getpass.getpass("External Redis URL (input hidden): ").strip()
mail_choices = (
("disabled", "external-relay")
if args.profile == "self-hosted"
@@ -599,20 +699,43 @@ def _prompt_configuration(args: argparse.Namespace) -> None:
args.mail,
mail_choices,
)
args.storage = _prompt_choice("File storage", args.storage, ("local", "s3"))
args.storage = _prompt_choice(
"File storage",
args.storage,
("local", "garage", "s3"),
)
if args.storage == "s3":
args.s3_endpoint_url = args.s3_endpoint_url or _prompt(
"S3 endpoint URL", "https://s3.example.org"
)
args.s3_region = args.s3_region or _prompt("S3 region", "eu-central-1")
args.s3_access_key_id = args.s3_access_key_id or _prompt(
"S3 access key id", ""
)
args.s3_access_key_id = args.s3_access_key_id or _prompt("S3 access key id", "")
args.s3_secret_access_key = (
args.s3_secret_access_key
or getpass.getpass("S3 secret access key (input hidden): ").strip()
)
args.s3_bucket = args.s3_bucket or _prompt("S3 bucket", "govoplan-files")
args.api_replicas = _prompt_integer(
"API replicas on this host",
args.api_replicas,
minimum=1,
maximum=64,
)
args.web_replicas = _prompt_integer(
"WebUI replicas on this host",
args.web_replicas,
minimum=1,
maximum=64,
)
if args.redis != "disabled":
args.worker_replicas = _prompt_integer(
"Worker replicas on this host",
args.worker_replicas or 1,
minimum=1,
maximum=128,
)
else:
args.worker_replicas = 0
args.module_set = _prompt_choice(
"Initial module set", args.module_set, ("core", "base", "full")
)
@@ -631,6 +754,24 @@ def _prompt_choice(label: str, default: str, choices: tuple[str, ...]) -> str:
print(f"Choose one of: {', '.join(choices)}")
def _prompt_integer(
label: str,
default: int,
*,
minimum: int,
maximum: int,
) -> int:
while True:
raw = _prompt(label, str(default))
try:
value = int(raw)
except ValueError:
value = minimum - 1
if minimum <= value <= maximum:
return value
print(f"Choose a number between {minimum} and {maximum}.")
@contextmanager
def _deployment_lock(path: Path) -> Iterator[None]:
descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
@@ -638,7 +779,9 @@ def _deployment_lock(path: Path) -> Iterator[None]:
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise ValueError("another deployment operation owns the installation lock") from exc
raise ValueError(
"another deployment operation owns the installation lock"
) from exc
os.ftruncate(descriptor, 0)
os.write(descriptor, f"{os.getpid()}\n".encode("ascii"))
os.fsync(descriptor)
@@ -686,7 +829,22 @@ def _parse_compose_ps(output: str) -> list[object]:
return value if isinstance(value, list) else [value]
def _receipt_uses_direct_web_port(path: Path) -> bool:
if not path.exists():
return False
try:
receipt = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return False
services = receipt.get("services") if isinstance(receipt, dict) else None
return (
isinstance(services, list)
and "web" in services
and "load-balancer" not in services
)
def _now() -> str:
return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace(
"+00:00", "Z"
return (
datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
)
tools/deployment/govoplan_deploy/model.py
+120 -17
View File
@@ -12,6 +12,8 @@ from urllib.parse import urlsplit
SCHEMA_VERSION = 1
DEFAULT_GARAGE_IMAGE = "dxflrs/garage:v2.3.0"
DEFAULT_LOAD_BALANCER_IMAGE = "haproxy:3.2.21-alpine"
INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$")
ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$")
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
@@ -87,6 +89,7 @@ class ServiceConfig:
@dataclass(frozen=True, slots=True)
class StorageConfig:
mode: str
image: str = ""
@dataclass(frozen=True, slots=True)
@@ -95,6 +98,14 @@ class ComponentConfig:
redis: ServiceConfig
mail: ServiceConfig
storage: StorageConfig
load_balancer: ServiceConfig
@dataclass(frozen=True, slots=True)
class ReplicaConfig:
api: int
web: int
worker: int
@dataclass(frozen=True, slots=True)
@@ -107,6 +118,7 @@ class InstallationSpec:
network_subnet: str
release: ReleaseConfig
components: ComponentConfig
replicas: ReplicaConfig
enabled_modules: tuple[str, ...]
def to_dict(self) -> dict[str, Any]:
@@ -126,6 +138,11 @@ def default_spec(
redis_mode: str = "managed",
mail_mode: str = "disabled",
storage_mode: str = "local",
garage_image: str = DEFAULT_GARAGE_IMAGE,
load_balancer_image: str = DEFAULT_LOAD_BALANCER_IMAGE,
api_replicas: int = 1,
web_replicas: int = 1,
worker_replicas: int | None = None,
module_set: str = "base",
api_image: str = "govoplan-api:unpublished",
web_image: str = "govoplan-web:unpublished",
@@ -141,6 +158,11 @@ def default_spec(
}.get(module_set)
if modules is None:
raise SpecError("module_set must be core, base, or full")
effective_worker_replicas = (
(0 if redis_mode == "disabled" else 1)
if worker_replicas is None
else worker_replicas
)
subnet_octet = 32 + (sum(installation_id.encode("utf-8")) % 192)
raw = {
"schema_version": SCHEMA_VERSION,
@@ -173,7 +195,20 @@ def default_spec(
"image": "greenmail/standalone:2.1.9",
"url_env": "",
},
"storage": {"mode": storage_mode},
"storage": {
"mode": storage_mode,
"image": garage_image if storage_mode == "garage" else "",
},
"load_balancer": {
"mode": "managed",
"image": load_balancer_image,
"url_env": "",
},
},
"replicas": {
"api": api_replicas,
"web": web_replicas,
"worker": effective_worker_replicas,
},
"enabled_modules": list(modules),
}
@@ -203,6 +238,7 @@ def parse_spec(raw: object) -> InstallationSpec:
"network_subnet",
"release",
"components",
"replicas",
"enabled_modules",
},
"installation",
@@ -233,11 +269,10 @@ def parse_spec(raw: object) -> InstallationSpec:
port=_port(_integer(listen_raw, "port"), "listen.port"),
)
network_subnet = _network(
_string(root, "network_subnet"), "network_subnet"
)
network_subnet = _network(_string(root, "network_subnet"), "network_subnet")
release = _release(root.get("release"))
components = _components(root.get("components"), profile=profile)
replicas = _replicas(root.get("replicas"), components=components)
enabled_raw = root.get("enabled_modules")
if not isinstance(enabled_raw, list):
@@ -248,9 +283,7 @@ def parse_spec(raw: object) -> InstallationSpec:
if not isinstance(value, str) or not re.fullmatch(
r"[a-z][a-z0-9_]{1,63}", value
):
raise SpecError(
f"enabled_modules[{index}] must be a canonical module id"
)
raise SpecError(f"enabled_modules[{index}] must be a canonical module id")
if value in seen_modules:
raise SpecError(f"enabled_modules contains duplicate module {value!r}")
enabled_modules.append(value)
@@ -265,6 +298,7 @@ def parse_spec(raw: object) -> InstallationSpec:
network_subnet=network_subnet,
release=release,
components=components,
replicas=replicas,
enabled_modules=tuple(enabled_modules),
)
@@ -296,7 +330,9 @@ def _release(raw: object) -> ReleaseConfig:
raise SpecError("release.manifest_url must use HTTPS")
manifest_sha256 = _optional_string(value, "manifest_sha256").lower()
if manifest_sha256 and not SHA256_PATTERN.fullmatch(manifest_sha256):
raise SpecError("release.manifest_sha256 must be a lowercase SHA-256 hex digest")
raise SpecError(
"release.manifest_sha256 must be a lowercase SHA-256 hex digest"
)
api_image = _image(_string(value, "api_image"), "release.api_image")
web_image = _image(_string(value, "web_image"), "release.web_image")
return ReleaseConfig(
@@ -311,7 +347,11 @@ def _release(raw: object) -> ReleaseConfig:
def _components(raw: object, *, profile: str) -> ComponentConfig:
value = _mapping(raw, "components")
_only_keys(value, {"postgres", "redis", "mail", "storage"}, "components")
_only_keys(
value,
{"postgres", "redis", "mail", "storage", "load_balancer"},
"components",
)
postgres = _service(
value.get("postgres"),
"components.postgres",
@@ -334,14 +374,41 @@ def _components(raw: object, *, profile: str) -> ComponentConfig:
url_env_required_for=set(),
)
storage_raw = _mapping(value.get("storage"), "components.storage")
_only_keys(storage_raw, {"mode"}, "components.storage")
storage = StorageConfig(mode=_choice(storage_raw, "mode", {"local", "s3"}))
_only_keys(storage_raw, {"mode", "image"}, "components.storage")
storage_mode = _choice(storage_raw, "mode", {"local", "s3", "garage"})
storage_image = _optional_string(storage_raw, "image")
if storage_mode == "garage":
storage_image = _image(
storage_image or DEFAULT_GARAGE_IMAGE,
"components.storage.image",
)
elif storage_image:
raise SpecError(
"components.storage.image is only valid for managed Garage storage"
)
storage = StorageConfig(mode=storage_mode, image=storage_image)
load_balancer = _service(
value.get(
"load_balancer",
{
"mode": "managed",
"image": DEFAULT_LOAD_BALANCER_IMAGE,
"url_env": "",
},
),
"components.load_balancer",
modes={"managed"},
image_required_for={"managed"},
url_env_required_for=set(),
)
if postgres.url_env != "DATABASE_URL":
raise SpecError("components.postgres.url_env must be DATABASE_URL")
if redis.url_env != "REDIS_URL":
raise SpecError("components.redis.url_env must be REDIS_URL")
if mail.url_env:
raise SpecError("components.mail.url_env must be empty")
if load_balancer.url_env:
raise SpecError("components.load_balancer.url_env must be empty")
if profile == "self-hosted" and redis.mode == "disabled":
raise SpecError("self-hosted installations require managed or external Redis")
if profile == "self-hosted" and mail.mode == "test-mail":
@@ -351,9 +418,36 @@ def _components(raw: object, *, profile: str) -> ComponentConfig:
redis=redis,
mail=mail,
storage=storage,
load_balancer=load_balancer,
)
def _replicas(raw: object, *, components: ComponentConfig) -> ReplicaConfig:
if raw is None:
return ReplicaConfig(
api=1,
web=1,
worker=0 if components.redis.mode == "disabled" else 1,
)
value = _mapping(raw, "replicas")
_only_keys(value, {"api", "web", "worker"}, "replicas")
replicas = ReplicaConfig(
api=_bounded_integer(value, "api", minimum=1, maximum=64),
web=_bounded_integer(value, "web", minimum=1, maximum=64),
worker=_bounded_integer(value, "worker", minimum=0, maximum=128),
)
if components.redis.mode == "disabled":
if replicas.worker:
raise SpecError("replicas.worker must be 0 when Redis is disabled")
if replicas.api > 1:
raise SpecError(
"multiple API replicas require Redis for shared throttling and queues"
)
elif replicas.worker < 1:
raise SpecError("replicas.worker must be at least 1 when Redis is enabled")
return replicas
def _service(
raw: object,
label: str,
@@ -418,6 +512,19 @@ def _integer(value: Mapping[str, Any], key: str) -> int:
return result
def _bounded_integer(
value: Mapping[str, Any],
key: str,
*,
minimum: int,
maximum: int,
) -> int:
result = _integer(value, key)
if result < minimum or result > maximum:
raise SpecError(f"{key} must be between {minimum} and {maximum}")
return result
def _choice(value: Mapping[str, Any], key: str, choices: set[str]) -> str:
result = _string(value, key)
if result not in choices:
@@ -434,9 +541,7 @@ def _http_url(value: str, label: str) -> str:
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise SpecError(f"{label} must be an absolute HTTP(S) URL")
if parsed.username or parsed.password or parsed.fragment or parsed.query:
raise SpecError(
f"{label} must not contain credentials, a query, or a fragment"
)
raise SpecError(f"{label} must not contain credentials, a query, or a fragment")
if label == "public_url" and parsed.path not in {"", "/"}:
raise SpecError("public_url must address the site root without a path")
return value.rstrip("/")
@@ -470,9 +575,7 @@ def _network(value: str, label: str) -> str:
or not any(network.subnet_of(parent) for parent in RFC1918_NETWORKS)
or not 24 <= network.prefixlen <= 28
):
raise SpecError(
f"{label} must be an RFC1918 IPv4 /24 to /28 network"
)
raise SpecError(f"{label} must be an RFC1918 IPv4 /24 to /28 network")
return str(network)
tools/deployment/govoplan_deploy/planning.py
+56 -9
View File
@@ -68,9 +68,7 @@ class DeploymentPlan:
"installation_id": self.installation_id,
"desired_spec_sha256": self.desired_spec_sha256,
"desired_compose_sha256": self.desired_compose_sha256,
"desired_environment_fingerprint": (
self.desired_environment_fingerprint
),
"desired_environment_fingerprint": (self.desired_environment_fingerprint),
"blocked": self.blocked,
"actions": [action.to_dict() for action in self.actions],
"checks": [check.to_dict() for check in self.checks],
@@ -151,7 +149,9 @@ def build_plan(
and previous_services == desired_services
):
actions.append(
PlanAction("noop", "installation", "Desired state matches the last receipt.")
PlanAction(
"noop", "installation", "Desired state matches the last receipt."
)
)
checks = list(static_checks(spec, paths))
@@ -179,6 +179,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
images["components.redis.image"] = spec.components.redis.image
if spec.components.mail.mode == "test-mail":
images["components.mail.image"] = spec.components.mail.image
if spec.components.storage.mode == "garage":
images["components.storage.image"] = spec.components.storage.image
images["components.load_balancer.image"] = spec.components.load_balancer.image
for label, image in images.items():
if image_is_unpublished(image):
@@ -248,7 +251,7 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
required = {"MASTER_KEY_B64", "DATABASE_URL"}
if spec.components.redis.mode != "disabled":
required.add("REDIS_URL")
if spec.components.storage.mode == "s3":
if spec.components.storage.mode in {"s3", "garage"}:
required.update(
{
"FILE_STORAGE_S3_ENDPOINT_URL",
@@ -258,6 +261,17 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
"FILE_STORAGE_S3_BUCKET",
}
)
if spec.components.storage.mode == "garage":
required.update(
{
"GARAGE_DEFAULT_ACCESS_KEY",
"GARAGE_DEFAULT_SECRET_KEY",
"GARAGE_DEFAULT_BUCKET",
"GARAGE_RPC_SECRET",
"GARAGE_ADMIN_TOKEN",
"GARAGE_METRICS_TOKEN",
}
)
missing = sorted(name for name in required if not values.get(name))
checks.append(
Check(
@@ -268,7 +282,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
if missing
else "Required runtime secret references are populated."
),
"Re-run configure with the required external service values." if missing else "",
"Re-run configure with the required external service values."
if missing
else "",
)
)
if paths.env.exists():
@@ -312,6 +328,39 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ..
"Include files-data in backup/restore drills or configure S3 storage.",
)
)
if spec.components.storage.mode == "garage":
checks.append(
Check(
"storage.garage.single_node",
"warning",
(
"Managed Garage is persistent S3-compatible storage, but "
"this Compose profile runs one Garage node without data redundancy."
),
(
"Use an external multi-node Garage/S3 service and tested "
"backup/restore for an availability-sensitive installation."
),
)
)
checks.append(
Check(
"topology.load_balancing",
"ok",
(
"Managed HAProxy balances "
f"{spec.replicas.web} WebUI and {spec.replicas.api} API replica(s)."
),
)
)
if spec.replicas.worker > 1:
checks.append(
Check(
"topology.worker_scaling",
"ok",
f"{spec.replicas.worker} workers share the configured Redis queues.",
)
)
return tuple(checks)
@@ -566,9 +615,7 @@ def _endpoint_check(
)
def _run_command(
argv: Sequence[str], cwd: Path
) -> subprocess.CompletedProcess[str]:
def _run_command(argv: Sequence[str], cwd: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
list(argv),
cwd=cwd,