GovOPlaN/govoplan
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add scalable deployment planning and guided releases
Security Audit / security-audit (push) Failing after 4s
Details
Dependency Audit / dependency-audit (push) Failing after 7s
Details
Deployment Installer / deployment-installer (push) Failing after 4s
Details

Browse Source
This commit is contained in:
zemion
2026-07-31 02:49:03 +02:00
parent 3864ce28b1
commit 908090dd0f
13 changed files with 1511 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/deployment/govoplan_deploy/bundle.py
+188 -22
View File
@@ -20,6 +20,8 @@ from .model import ENV_NAME_PATTERN, InstallationSpec
SPEC_FILENAME = "installation.json"
ENV_FILENAME = "secrets.env"
COMPOSE_FILENAME = "compose.json"
GARAGE_CONFIG_FILENAME = "garage.toml"
LOAD_BALANCER_CONFIG_FILENAME = "load-balancer.cfg"
PLAN_FILENAME = "plan.json"
RECEIPT_FILENAME = "receipt.json"
LOCK_FILENAME = ".deployment.lock"
@@ -52,6 +54,7 @@ RUNTIME_ENV_KEYS = (
"FILE_STORAGE_S3_ACCESS_KEY_ID",
"FILE_STORAGE_S3_SECRET_ACCESS_KEY",
"FILE_STORAGE_S3_BUCKET",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
)
@@ -61,6 +64,8 @@ class BundlePaths:
spec: Path
env: Path
compose: Path
garage_config: Path
load_balancer_config: Path
plan: Path
receipt: Path
lock: Path
@@ -69,15 +74,15 @@ class BundlePaths:
def bundle_paths(root: Path) -> BundlePaths:
expanded = root.expanduser()
if expanded.is_symlink():
raise ValueError(
f"installation root must not be a symbolic link: {expanded}"
)
raise ValueError(f"installation root must not be a symbolic link: {expanded}")
resolved = expanded.resolve()
return BundlePaths(
root=resolved,
spec=resolved / SPEC_FILENAME,
env=resolved / ENV_FILENAME,
compose=resolved / COMPOSE_FILENAME,
garage_config=resolved / GARAGE_CONFIG_FILENAME,
load_balancer_config=resolved / LOAD_BALANCER_CONFIG_FILENAME,
plan=resolved / PLAN_FILENAME,
receipt=resolved / RECEIPT_FILENAME,
lock=resolved / LOCK_FILENAME,
@@ -190,10 +195,37 @@ def reconcile_runtime_environment(
values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false"
storage = spec.components.storage
values["FILE_STORAGE_BACKEND"] = storage.mode
if storage.mode == "local":
values["FILE_STORAGE_BACKEND"] = "local"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files"
elif storage.mode == "garage":
access_key = values.setdefault(
"GARAGE_DEFAULT_ACCESS_KEY",
f"GK{secrets.token_hex(16)}",
)
secret_key = values.setdefault(
"GARAGE_DEFAULT_SECRET_KEY",
secrets.token_hex(32),
)
bucket = values.setdefault("GARAGE_DEFAULT_BUCKET", "govoplan-files")
values.setdefault("GARAGE_RPC_SECRET", secrets.token_hex(32))
values.setdefault("GARAGE_ADMIN_TOKEN", secrets.token_urlsafe(48))
values.setdefault("GARAGE_METRICS_TOKEN", secrets.token_urlsafe(48))
values.update(
{
"FILE_STORAGE_BACKEND": "s3",
"FILE_STORAGE_S3_ENDPOINT_URL": "http://garage:3900",
"FILE_STORAGE_S3_REGION": "garage",
"FILE_STORAGE_S3_ACCESS_KEY_ID": access_key,
"FILE_STORAGE_S3_SECRET_ACCESS_KEY": secret_key,
"FILE_STORAGE_S3_BUCKET": bucket,
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "true",
}
)
else:
values["FILE_STORAGE_BACKEND"] = "s3"
values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false"
required = (
"FILE_STORAGE_S3_ENDPOINT_URL",
"FILE_STORAGE_S3_REGION",
@@ -278,6 +310,50 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
dependency_conditions["redis"] = {"condition": "service_healthy"}
if spec.components.storage.mode == "garage":
garage_environment = _environment_references(
(
"GARAGE_DEFAULT_ACCESS_KEY",
"GARAGE_DEFAULT_SECRET_KEY",
"GARAGE_DEFAULT_BUCKET",
"GARAGE_RPC_SECRET",
"GARAGE_ADMIN_TOKEN",
"GARAGE_METRICS_TOKEN",
),
required=True,
)
services["garage"] = {
"image": spec.components.storage.image,
"restart": "unless-stopped",
"command": [
"/garage",
"server",
"--single-node",
"--default-bucket",
],
"environment": garage_environment,
"healthcheck": {
"test": ["CMD", "/garage", "status"],
"interval": "10s",
"timeout": "5s",
"retries": 30,
"start_period": "15s",
},
"labels": {
"org.govoplan.configuration-sha256": sha256(
render_garage_config().encode("utf-8")
).hexdigest()
},
"security_opt": ["no-new-privileges:true"],
"volumes": [
f"./{GARAGE_CONFIG_FILENAME}:/etc/garage.toml:ro",
"garage-meta:/var/lib/garage/meta",
"garage-data:/var/lib/garage/data",
],
"networks": ["internal"],
}
dependency_conditions["garage"] = {"condition": "service_healthy"}
if spec.components.mail.mode == "test-mail":
services["test-mail"] = {
"image": spec.components.mail.image,
@@ -312,6 +388,7 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
services["api"] = {
**common_runtime,
"image": spec.release.api_image,
"scale": spec.replicas.api,
"command": [
"python",
"-m",
@@ -342,15 +419,43 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
services["web"] = {
"image": spec.release.web_image,
"restart": "unless-stopped",
"environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"},
"depends_on": {"api": {"condition": "service_healthy"}},
"scale": spec.replicas.web,
"environment": {"GOVOPLAN_API_UPSTREAM": "http://load-balancer:8000"},
"networks": ["internal"],
}
services["load-balancer"] = {
"image": spec.components.load_balancer.image,
"restart": "unless-stopped",
"healthcheck": {
"test": [
"CMD",
"haproxy",
"-c",
"-f",
"/usr/local/etc/haproxy/haproxy.cfg",
],
"interval": "10s",
"timeout": "5s",
"retries": 10,
},
"labels": {
"org.govoplan.configuration-sha256": sha256(
render_load_balancer_config(spec).encode("utf-8")
).hexdigest()
},
"ports": [_published_port(spec.listen.address, spec.listen.port, 8080)],
"read_only": True,
"security_opt": ["no-new-privileges:true"],
"volumes": [
(f"./{LOAD_BALANCER_CONFIG_FILENAME}:/usr/local/etc/haproxy/haproxy.cfg:ro")
],
"networks": ["internal"],
}
if spec.components.redis.mode != "disabled":
services["worker"] = {
**common_runtime,
"image": spec.release.api_image,
"scale": spec.replicas.worker,
"command": [
"python",
"-m",
@@ -388,6 +493,9 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
volumes["redis-data"] = {}
if spec.components.storage.mode == "local":
volumes["files-data"] = {}
if spec.components.storage.mode == "garage":
volumes["garage-meta"] = {}
volumes["garage-data"] = {}
return {
"name": spec.installation_id,
@@ -402,14 +510,81 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
}
def render_garage_config() -> str:
return """metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.localhost"
[admin]
api_bind_addr = "[::]:3903"
"""
def render_load_balancer_config(spec: InstallationSpec) -> str:
return f"""global
log stdout format raw local0
maxconn 4096
defaults
log global
mode http
option httplog
option redispatch
timeout connect 5s
timeout client 60s
timeout server 60s
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold other 10s
hold refused 10s
hold nx 10s
hold timeout 10s
hold valid 10s
hold obsolete 10s
frontend public_web
bind :8080
default_backend web_replicas
backend web_replicas
balance roundrobin
option httpchk GET /health
http-check expect status 200
server-template web- {spec.replicas.web} web:8080 check resolvers docker init-addr libc,none
frontend internal_api
bind :8000
default_backend api_replicas
backend api_replicas
balance leastconn
option httpchk GET /health
http-check expect status 200
server-template api- {spec.replicas.api} api:8000 check resolvers docker init-addr libc,none
"""
def service_names(spec: InstallationSpec) -> tuple[str, ...]:
return tuple(render_compose(spec)["services"].keys())
def canonical_json(value: object) -> bytes:
return (
json.dumps(value, indent=2, sort_keys=True, separators=(",", ": "))
+ "\n"
json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + "\n"
).encode("utf-8")
@@ -439,9 +614,7 @@ def read_env(path: Path) -> dict[str, str]:
if not separator or not ENV_NAME_PATTERN.fullmatch(key):
raise ValueError(f"invalid environment line {line_number} in {path}")
if key in values:
raise ValueError(
f"duplicate environment key {key!r} on line {line_number}"
)
raise ValueError(f"duplicate environment key {key!r} on line {line_number}")
value = raw_value
if value.startswith('"'):
try:
@@ -464,9 +637,7 @@ def read_env(path: Path) -> dict[str, str]:
def write_env(path: Path, values: Mapping[str, str]) -> None:
invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key))
if invalid:
raise ValueError(
"invalid environment variable names: " + ", ".join(invalid)
)
raise ValueError("invalid environment variable names: " + ", ".join(invalid))
lines = [
"# Generated by govoplan-deploy. Keep this file private.",
*[f"{key}={_env_value(value)}" for key, value in sorted(values.items())],
@@ -477,9 +648,7 @@ def write_env(path: Path, values: Mapping[str, str]) -> None:
def atomic_write(path: Path, payload: bytes, *, mode: int) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
temporary = path.with_name(
f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
)
temporary = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp")
descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
@@ -506,8 +675,7 @@ def _env_value(value: str) -> str:
if "\x00" in value or "\n" in value or "\r" in value:
raise ValueError("environment values must not contain NUL or line breaks")
if value and all(
character.isalnum() or character in "_./:@%+,-"
for character in value
character.isalnum() or character in "_./:@%+,-" for character in value
):
return value
escaped = value.replace("\\", "\\\\").replace("'", "\\'")
@@ -550,9 +718,7 @@ def _single_quoted_env_value(value: str, *, line_number: int) -> str:
continue
index += 1
if index >= len(body) or body[index] not in {"\\", "'"}:
raise ValueError(
f"invalid single-quoted escape on line {line_number}"
)
raise ValueError(f"invalid single-quoted escape on line {line_number}")
output.append(body[index])
index += 1
return "".join(output)