diff --git a/.gitea/workflows/deployment-installer.yml b/.gitea/workflows/deployment-installer.yml new file mode 100644 index 0000000..ee781fa --- /dev/null +++ b/.gitea/workflows/deployment-installer.yml @@ -0,0 +1,30 @@ +name: Deployment Installer + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +jobs: + deployment-installer: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + path: govoplan + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Compile deployment tooling + working-directory: govoplan + run: python -m py_compile tools/deployment/govoplan-deploy.py tools/deployment/govoplan_deploy/*.py + - name: Test declarative deployment bundle + working-directory: govoplan + run: python -m unittest -v tests.test_deployment_installer + - name: Build single-file deployer artifact + working-directory: govoplan + run: | + python tools/deployment/build-deployer-zipapp.py --output /tmp/govoplan-deploy.pyz + python /tmp/govoplan-deploy.pyz --help diff --git a/README.md b/README.md index 2f87b02..5f49a9e 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Module Matrix](https://git.add-ideas.de/GovOPlaN/govoplan/actions/workflows/module-matrix.yml/badge.svg?branch=main)](https://git.add-ideas.de/GovOPlaN/govoplan/actions?workflow=module-matrix.yml&actor=0&status=0) [![Release Integration](https://git.add-ideas.de/GovOPlaN/govoplan/actions/workflows/release-integration.yml/badge.svg?branch=main)](https://git.add-ideas.de/GovOPlaN/govoplan/actions?workflow=release-integration.yml&actor=0&status=0) +[![Deployment Installer](https://git.add-ideas.de/GovOPlaN/govoplan/actions/workflows/deployment-installer.yml/badge.svg?branch=main)](https://git.add-ideas.de/GovOPlaN/govoplan/actions?workflow=deployment-installer.yml&actor=0&status=0) [![Dependency Audit](https://git.add-ideas.de/GovOPlaN/govoplan/actions/workflows/dependency-audit.yml/badge.svg?branch=main)](https://git.add-ideas.de/GovOPlaN/govoplan/actions?workflow=dependency-audit.yml&actor=0&status=0) [![Security Audit](https://git.add-ideas.de/GovOPlaN/govoplan/actions/workflows/security-audit.yml/badge.svg?branch=main)](https://git.add-ideas.de/GovOPlaN/govoplan/actions?workflow=security-audit.yml&actor=0&status=0) @@ -141,6 +142,18 @@ Start the local release console: ./.venv/bin/python tools/release/release-console.py ``` +Create and validate a private, declarative installation bundle: + +```sh +./.venv/bin/python tools/deployment/govoplan-deploy.py init \ + --directory ~/.local/share/govoplan/installations/default +./.venv/bin/python tools/deployment/govoplan-deploy.py doctor \ + --directory ~/.local/share/govoplan/installations/default +``` + +The current executable slice and remaining production gates are documented in +[Installation and Deployment Architecture](docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md). + ## Configuration The repository root `.env.example` is the self-hosted operator template for a @@ -173,6 +186,9 @@ including stage gates and shared documentation expectations, is in the The administrator journey from Core-only bootstrap through online module installation, scale-out, and reversible environment promotion is defined in [System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md). +The corresponding host deployment compiler, managed/external component choices, +reconfiguration semantics, and safe Web update boundary are defined in +[Installation and Deployment Architecture](docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md). The first Campaign-centric capability and infrastructure fit assessment is in `docs/CAPABILITY_AND_INFRASTRUCTURE_FIT.md`. Its rerun tooling can collect and verify a bounded installed composition; target, provider and production claims diff --git a/docker/README.md b/docker/README.md index ebe2ffd..3d42b9f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,6 +7,11 @@ Current shared profiles: - `govoplan/dev/postgres` - `govoplan/dev/production-like` +The generated whole-product Compose profile is owned by +`tools/deployment/govoplan-deploy.py`. It renders a deployment-specific +`compose.json` from a versioned installation specification; generated files and +secrets remain outside the repository. + Module-specific Docker test beds remain in their owning repositories: - `govoplan-campaign/dev/mail-testbed` diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md new file mode 100644 index 0000000..3718615 --- /dev/null +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -0,0 +1,267 @@ +# Installation And Deployment Architecture + +## Goal + +A supported GovOPlaN installation starts with one downloaded, verified +bootstrap artifact. The administrator answers a bounded set of questions and +receives a working base system. Re-running the same tool repairs or +reconfigures that installation instead of creating unrelated state. + +The canonical product journey remains +[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md). +This document defines the deployer boundary and the first executable slice. + +## First Executable Slice + +`tools/deployment/govoplan-deploy.py` is a standard-library-only deployment +compiler and reconciler. It can be tested without installing GovOPlaN itself. + +It currently supports: + +- evaluation and self-hosted profiles; +- managed or external PostgreSQL; +- managed, external, or evaluation-only disabled Redis; +- disabled mail, an external relay declaration, or an evaluation-only + GreenMail service; +- durable local file storage or external S3-compatible storage; +- Core, base, or full initial module selections; +- deterministic Compose JSON accepted by Compose v2; +- generated secrets stored in a private `0600` file; +- service-specific environment allowlists so infrastructure containers do not + receive unrelated application credentials; +- plan, render, doctor, status, and apply commands; +- an installation lock, migration-before-start ordering, readiness polling, + and an applied-state receipt; +- idempotent reconfiguration that preserves generated secrets; +- a keyed environment fingerprint that detects private binding changes without + writing secret values to plans or receipts; +- host CPU, memory, disk, entropy, architecture, Docker daemon, Compose, + listen-port, and external endpoint preflight checks; +- service removal without implicit data-volume deletion. + +Create a local evaluation bundle: + +```sh +./.venv/bin/python tools/deployment/govoplan-deploy.py init \ + --directory /tmp/govoplan-evaluation \ + --profile evaluation \ + --postgres managed \ + --redis managed \ + --mail test-mail \ + --module-set base +``` + +Inspect the generated intent and host requirements: + +```sh +./.venv/bin/python tools/deployment/govoplan-deploy.py doctor \ + --directory /tmp/govoplan-evaluation +``` + +Change a component without rotating existing generated secrets: + +```sh +./.venv/bin/python tools/deployment/govoplan-deploy.py configure \ + --directory /tmp/govoplan-evaluation \ + --redis external \ + --redis-url 'rediss://:password@redis.example.org:6379/0' +``` + +The private installation directory contains: + +| File | Purpose | +| --- | --- | +| `installation.json` | Versioned, non-secret desired state | +| `secrets.env` | Deployment-local secrets and external service bindings | +| `compose.json` | Deterministic generated Compose definition | +| `plan.json` | Latest desired-state diff and readiness findings | +| `receipt.json` | Last successfully applied immutable identities | +| `.deployment.lock` | Same-host operation exclusion | + +The specification contract is +[`installation-spec.schema.json`](installation-spec.schema.json). + +Build the same dependency-free tool as one downloadable artifact: + +```sh +./.venv/bin/python tools/deployment/build-deployer-zipapp.py \ + --output /tmp/govoplan-deploy.pyz +python /tmp/govoplan-deploy.pyz --help +``` + +To exercise reconciliation with locally available evaluation images: + +```sh +python /tmp/govoplan-deploy.pyz init \ + --non-interactive \ + --directory /tmp/govoplan-evaluation \ + --profile evaluation \ + --api-image local/govoplan-api:test \ + --web-image local/govoplan-web:test +python /tmp/govoplan-deploy.pyz apply \ + --directory /tmp/govoplan-evaluation \ + --allow-unverified-images \ + --skip-pull +``` + +Those images must already contain the selected module set. The override exists +only to exercise local orchestration before release artifacts exist; it is +rejected for `self-hosted`. + +## Current Production Gates + +The tool deliberately reports blockers instead of pretending the source tree is +a production distribution: + +1. **OCI release artifacts.** The release pipeline does not yet publish pinned + multi-architecture API and WebUI images. +2. **Signed distribution manifest.** A channel manifest must bind exact image + digests, Compose compatibility, SBOM/provenance references, and revocation + state. Recording a URL and checksum is not signature verification. +3. **First administrator.** Production needs a one-time, restricted enrollment + identity. The development bootstrap must not be enabled in production. +4. **Image/module composition.** The selected module set must be proven present + in the exact image or installed from verified offline artifacts before it is + enabled. +5. **Deployment agent.** Web updates need a separate privileged reconciler with + a typed command allowlist. The API and browser must never receive the Docker + socket or arbitrary shell access. +6. **Ingress and certificates.** A self-hosted profile needs an explicit choice + between an existing reverse proxy and a supported managed ingress, including + trusted-proxy boundaries, TLS certificate issuance, renewal, and health + probing through the public route. + +`apply --allow-unverified-images` is therefore restricted to the evaluation +profile. It explicitly acknowledges both mutable image identities and +unverified image/module composition. It is a local test escape hatch, not a +production setting. + +## Component Choices + +### PostgreSQL + +`managed` creates a persistent PostgreSQL container and private generated +credentials. `external` requires an explicit `DATABASE_URL`; switching from +managed to external cannot reuse the old `postgres` Docker hostname +accidentally. + +Interactive entry hides external URLs because they commonly contain +credentials. For unattended automation, provide them through a protected +operator mechanism and avoid storing secret-bearing flags in shell history. + +Production policy should support external managed databases and local managed +PostgreSQL equally at the application boundary. Backup, point-in-time recovery, +high availability, and major-version upgrades remain deployment properties. + +### Redis + +`managed` creates an authenticated, append-only Redis container. `external` +requires an explicit `REDIS_URL`. `disabled` is evaluation-only and disables +workers while recording the single-process login-throttle risk acknowledgement. + +`doctor` performs a bounded TCP connection check for external PostgreSQL, +Redis, and S3 endpoints. This verifies DNS, routing, and that the port accepts a +connection; it is not an authentication or semantic health check. + +Production base installations include Redis because durable queues, distributed +throttling, notifications, scheduled work, and transactional event delivery +must survive API restarts. + +### Mail + +The first slice distinguishes: + +- `disabled`; +- `external-relay`, which records the infrastructure decision but leaves Mail + server/credential creation as a visible post-install task; +- `test-mail`, an evaluation-only GreenMail service. + +A bundled production mail server is intentionally not a default. Operating one +requires DNS, reverse DNS, TLS, DKIM, SPF, DMARC, reputation, abuse handling, +queue monitoring, and upgrade policy. A later profile may support an +operator-selected MTA/relay, but it must expose these requirements rather than +presenting a container as a complete mail service. + +### File Storage + +`local` uses a durable Compose volume and is appropriate for one-host +installations. `s3` requires endpoint, region, access key, secret key, and +bucket values. Self-hosted S3 endpoints must use HTTPS. + +Local storage must be included in backup and restore drills. Horizontal API or +worker scale-out requires shared/object storage. + +## Reconfiguration Semantics + +`installation.json` is desired state. `receipt.json` is the last successfully +applied state. `plan` compares their canonical hashes and service sets. + +- Adding a managed component creates its service and persistent volume. +- Removing a component removes its service container on apply. +- Volumes are retained by default; deleting data requires a separate, + deliberately destructive workflow. +- Existing generated credentials are retained unless an explicit future rotate + operation is requested. +- Private configuration changes are represented by a keyed fingerprint in the + plan and receipt; plaintext values are never copied there. +- Managed-to-external transitions require the new endpoint in the same + operation. +- Migrations run as a one-shot service before API/worker replacement. +- Health must recover before a new receipt is committed. + +This is sufficient for one-host reconciliation. Production updates additionally +need backup/restore gates, maintenance/drain state, database compatibility +windows, image signature verification, and rollback/forward-recovery policy. + +## Web Update Boundary + +The intended update path is: + +1. Ops reads the non-secret installation receipt and reports management mode, + current release, component health, and update availability. +2. An authorized administrator asks Core to create a typed deployment request, + for example `reconcile_release` or `rollback_release`. +3. Core persists the reviewed immutable plan, actor, expected current receipt, + and idempotency key. +4. A separately deployed, narrow deployment agent claims the request. +5. The agent verifies signatures/digests, acquires a fenced deployment lock, + backs up, pulls, migrates, reconciles, probes health, and writes evidence. +6. Ops presents durable progress and the resulting receipt. + +The agent owns container-runtime access. It accepts no command strings from the +browser and has no domain-data permissions. Installations managed by Kubernetes, +systemd, or another external orchestrator expose read-only status and an export +of the reviewed update recipe instead of a non-functional update button. + +## Distribution Workflow + +The downloadable entry point should eventually be: + +```sh +curl --proto '=https' --tlsv1.2 --fail --location \ + https://govoplan.add-ideas.de/install/v1/bootstrap.pyz \ + --output govoplan-bootstrap.pyz +python3 govoplan-bootstrap.pyz init +``` + +The published documentation must include an independent checksum/signature +verification command before execution. The zipapp then downloads only a signed +distribution manifest, verifies it against an embedded or explicitly installed +keyring, and renders the same installation contract implemented here. + +The source-tree script is the test harness for that future zipapp. It is not yet +the internet bootstrap artifact. + +## Verification + +Run the focused tests: + +```sh +./.venv/bin/python -m unittest -v tests.test_deployment_installer +``` + +The tests cover profile restrictions, secret persistence, external endpoint +requirements, S3 policy, Compose service selection, secret non-disclosure, +service-specific environment isolation, private file modes, external endpoint +preflight, first-plan generation, apply ordering, and receipt-based +idempotency. diff --git a/docs/installation-spec.schema.json b/docs/installation-spec.schema.json new file mode 100644 index 0000000..d00ef81 --- /dev/null +++ b/docs/installation-spec.schema.json @@ -0,0 +1,269 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://govoplan.add-ideas.de/schemas/installation-spec-v1.json", + "title": "GovOPlaN installation specification", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "installation_id", + "profile", + "public_url", + "listen", + "network_subnet", + "release", + "components", + "enabled_modules" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "installation_id": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{1,47}$" + }, + "profile": { + "enum": [ + "evaluation", + "self-hosted" + ] + }, + "public_url": { + "type": "string", + "format": "uri", + "pattern": "^https?://" + }, + "listen": { + "type": "object", + "additionalProperties": false, + "required": [ + "address", + "port" + ], + "properties": { + "address": { + "type": "string" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + } + }, + "network_subnet": { + "type": "string" + }, + "release": { + "type": "object", + "additionalProperties": false, + "required": [ + "channel", + "version", + "manifest_url", + "manifest_sha256", + "api_image", + "web_image" + ], + "properties": { + "channel": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{1,31}$" + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "manifest_url": { + "type": "string" + }, + "manifest_sha256": { + "type": "string", + "pattern": "^$|^[0-9a-f]{64}$" + }, + "api_image": { + "type": "string", + "minLength": 1, + "maxLength": 300 + }, + "web_image": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + } + }, + "components": { + "type": "object", + "additionalProperties": false, + "required": [ + "postgres", + "redis", + "mail", + "storage" + ], + "properties": { + "postgres": { + "$ref": "#/$defs/postgres" + }, + "redis": { + "$ref": "#/$defs/redis" + }, + "mail": { + "$ref": "#/$defs/mail" + }, + "storage": { + "type": "object", + "additionalProperties": false, + "required": [ + "mode" + ], + "properties": { + "mode": { + "enum": [ + "local", + "s3" + ] + } + } + } + } + }, + "enabled_modules": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{1,63}$" + } + } + }, + "$defs": { + "service": { + "type": "object", + "additionalProperties": false, + "required": [ + "mode", + "image", + "url_env" + ], + "properties": { + "mode": { + "type": "string" + }, + "image": { + "type": "string" + }, + "url_env": { + "type": "string" + } + } + }, + "postgres": { + "allOf": [ + { + "$ref": "#/$defs/service" + }, + { + "properties": { + "mode": { + "enum": [ + "managed", + "external" + ] + }, + "url_env": { + "const": "DATABASE_URL" + } + } + } + ] + }, + "redis": { + "allOf": [ + { + "$ref": "#/$defs/service" + }, + { + "properties": { + "mode": { + "enum": [ + "managed", + "external", + "disabled" + ] + }, + "url_env": { + "const": "REDIS_URL" + } + } + } + ] + }, + "mail": { + "allOf": [ + { + "$ref": "#/$defs/service" + }, + { + "properties": { + "mode": { + "enum": [ + "disabled", + "external-relay", + "test-mail" + ] + }, + "url_env": { + "const": "" + } + } + } + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "profile": { + "const": "self-hosted" + } + } + }, + "then": { + "properties": { + "public_url": { + "pattern": "^https://" + }, + "components": { + "properties": { + "redis": { + "properties": { + "mode": { + "enum": [ + "managed", + "external" + ] + } + } + }, + "mail": { + "properties": { + "mode": { + "enum": [ + "disabled", + "external-relay" + ] + } + } + } + } + } + } + } + } + ] +} diff --git a/tests/test_deployment_installer.py b/tests/test_deployment_installer.py new file mode 100644 index 0000000..171c5b6 --- /dev/null +++ b/tests/test_deployment_installer.py @@ -0,0 +1,644 @@ +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import io +import json +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + + +META_ROOT = Path(__file__).resolve().parents[1] +DEPLOYMENT_TOOLS = META_ROOT / "tools" / "deployment" +if str(DEPLOYMENT_TOOLS) not in sys.path: + sys.path.insert(0, str(DEPLOYMENT_TOOLS)) + +from govoplan_deploy.bundle import ( # noqa: E402 + atomic_write, + bundle_paths, + canonical_json, + environment_fingerprint, + initial_secrets, + read_env, + reconcile_runtime_environment, + render_compose, + write_env, +) +from govoplan_deploy.cli import main # noqa: E402 +import govoplan_deploy.cli as deployment_cli # noqa: E402 +from govoplan_deploy.model import ( # noqa: E402 + SpecError, + default_spec, + parse_spec, +) +from govoplan_deploy.planning import _endpoint_check, build_plan # noqa: E402 +import govoplan_deploy.planning as deployment_planning # noqa: E402 + + +def run_cli(arguments: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + result = main(arguments) + return result, stdout.getvalue(), stderr.getvalue() + + +class DeploymentInstallerTests(unittest.TestCase): + def test_default_bundle_has_base_modules_and_managed_dependencies(self) -> None: + spec = default_spec() + compose = render_compose(spec) + + self.assertEqual("evaluation", spec.profile) + self.assertIn("access", spec.enabled_modules) + self.assertIn("postgres", compose["services"]) + self.assertIn("redis", compose["services"]) + self.assertIn("worker", compose["services"]) + self.assertNotIn("test-mail", compose["services"]) + self.assertEqual( + ["127.0.0.1:8080:8080"], + compose["services"]["web"]["ports"], + ) + + def test_disabled_redis_removes_workers_and_sets_single_process_acknowledgement( + self, + ) -> None: + spec = default_spec(redis_mode="disabled") + values = initial_secrets(spec) + compose = render_compose(spec) + + self.assertNotIn("redis", compose["services"]) + self.assertNotIn("worker", compose["services"]) + self.assertNotIn("scheduler", compose["services"]) + self.assertEqual("false", values["CELERY_ENABLED"]) + self.assertEqual( + "true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] + ) + + def test_self_hosted_rejects_insecure_or_test_only_choices(self) -> None: + with self.assertRaisesRegex(SpecError, "must use HTTPS"): + default_spec(profile="self-hosted") + with self.assertRaisesRegex(SpecError, "require managed or external Redis"): + default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + redis_mode="disabled", + ) + with self.assertRaisesRegex(SpecError, "test-mail"): + default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + mail_mode="test-mail", + ) + + def test_spec_rejects_unknown_fields_and_duplicate_modules(self) -> None: + raw = default_spec().to_dict() + raw["unexpected"] = True + with self.assertRaisesRegex(SpecError, "unknown fields"): + parse_spec(raw) + + raw = default_spec().to_dict() + raw["enabled_modules"].append(raw["enabled_modules"][0]) + with self.assertRaisesRegex(SpecError, "duplicate module"): + parse_spec(raw) + + raw = default_spec().to_dict() + raw["network_subnet"] = "127.0.0.0/24" + with self.assertRaisesRegex(SpecError, "RFC1918"): + parse_spec(raw) + + def test_generated_secrets_are_stable_across_reconfiguration(self) -> None: + first = default_spec() + values = initial_secrets(first) + master_key = values["MASTER_KEY_B64"] + postgres_password = values["POSTGRES_PASSWORD"] + redis_password = values["REDIS_PASSWORD"] + + second = default_spec(mail_mode="test-mail", module_set="full") + reconciled = reconcile_runtime_environment(second, values) + + self.assertEqual(master_key, reconciled["MASTER_KEY_B64"]) + self.assertEqual(postgres_password, reconciled["POSTGRES_PASSWORD"]) + self.assertEqual(redis_password, reconciled["REDIS_PASSWORD"]) + self.assertEqual( + ",".join(second.enabled_modules), reconciled["ENABLED_MODULES"] + ) + + def test_external_services_require_explicit_valid_urls(self) -> None: + spec = default_spec(postgres_mode="external", redis_mode="external") + with self.assertRaisesRegex(ValueError, "external PostgreSQL"): + initial_secrets(spec) + with self.assertRaisesRegex(ValueError, "external Redis"): + initial_secrets( + spec, + supplied={ + "DATABASE_URL": ( + "postgresql+psycopg://user:secret@db.example.test/govoplan" + ) + }, + ) + values = initial_secrets( + spec, + supplied={ + "DATABASE_URL": ( + "postgresql+psycopg://user:secret@db.example.test/govoplan" + ), + "REDIS_URL": "rediss://:secret@redis.example.test/0", + }, + ) + self.assertEqual( + "rediss://:secret@redis.example.test/0", values["REDIS_URL"] + ) + + def test_s3_requires_complete_private_configuration_and_https_in_production( + self, + ) -> None: + evaluation = default_spec(storage_mode="s3") + with self.assertRaisesRegex(ValueError, "S3 storage requires"): + initial_secrets(evaluation) + + supplied = { + "FILE_STORAGE_S3_ENDPOINT_URL": "http://s3.example.test", + "FILE_STORAGE_S3_REGION": "eu-test-1", + "FILE_STORAGE_S3_ACCESS_KEY_ID": "key", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY": "secret", + "FILE_STORAGE_S3_BUCKET": "govoplan", + } + self.assertEqual( + "s3", + initial_secrets(evaluation, supplied=supplied)[ + "FILE_STORAGE_BACKEND" + ], + ) + production = default_spec( + profile="self-hosted", + public_url="https://govoplan.example.test", + storage_mode="s3", + ) + with self.assertRaisesRegex(ValueError, "must use HTTPS"): + initial_secrets(production, supplied=supplied) + + def test_compose_contains_no_secret_values(self) -> None: + spec = default_spec() + values = initial_secrets(spec) + rendered = json.dumps(render_compose(spec), sort_keys=True) + + for key in ( + "MASTER_KEY_B64", + "POSTGRES_PASSWORD", + "REDIS_PASSWORD", + ): + self.assertNotIn(values[key], rendered) + self.assertNotIn("secrets.env", rendered) + self.assertNotIn("env_file", rendered) + self.assertNotIn("./:/etc/govoplan", rendered) + self.assertNotIn("installer", render_compose(spec)["services"]) + postgres_environment = render_compose(spec)["services"]["postgres"][ + "environment" + ] + redis_environment = render_compose(spec)["services"]["redis"][ + "environment" + ] + self.assertEqual( + {"POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"}, + set(postgres_environment), + ) + self.assertEqual({"REDIS_PASSWORD"}, set(redis_environment)) + self.assertNotIn("MASTER_KEY_B64", postgres_environment) + self.assertNotIn("MASTER_KEY_B64", redis_environment) + + def test_secret_environment_round_trips_literal_special_characters(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + path = Path(directory) / "secrets.env" + values = { + "PASSWORD": r"dollar$ quote' slash\\ hash# space ", + "EMPTY_VALUE": "", + } + + write_env(path, values) + + self.assertEqual(values, read_env(path)) + + def test_first_plan_creates_services_and_applied_receipt_becomes_noop( + self, + ) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + paths = bundle_paths(Path(directory)) + paths.root.chmod(0o700) + spec = default_spec() + write_env(paths.env, initial_secrets(spec)) + first = build_plan(spec, paths, include_host_checks=False) + + self.assertIn( + ("create", "installation"), + {(action.action, action.target) for action in first.actions}, + ) + self.assertIn( + ("start", "api"), + {(action.action, action.target) for action in first.actions}, + ) + + receipt = { + "spec_sha256": first.desired_spec_sha256, + "compose_sha256": first.desired_compose_sha256, + "environment_fingerprint": ( + first.desired_environment_fingerprint + ), + "services": list(render_compose(spec)["services"]), + } + atomic_write(paths.receipt, canonical_json(receipt), mode=0o600) + second = build_plan(spec, paths, include_host_checks=False) + + self.assertEqual( + [("noop", "installation")], + [(action.action, action.target) for action in second.actions], + ) + + def test_reconfiguration_plans_removed_component_containers(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + paths = bundle_paths(Path(directory)) + paths.root.chmod(0o700) + first_spec = default_spec(mail_mode="test-mail") + first_environment = initial_secrets(first_spec) + write_env(paths.env, first_environment) + first_plan = build_plan( + first_spec, paths, include_host_checks=False + ) + atomic_write( + paths.receipt, + canonical_json( + { + "spec_sha256": first_plan.desired_spec_sha256, + "compose_sha256": first_plan.desired_compose_sha256, + "environment_fingerprint": ( + first_plan.desired_environment_fingerprint + ), + "services": list(render_compose(first_spec)["services"]), + } + ), + mode=0o600, + ) + + second_spec = default_spec(redis_mode="disabled", mail_mode="disabled") + write_env( + paths.env, + reconcile_runtime_environment(second_spec, first_environment), + ) + second_plan = build_plan( + second_spec, paths, include_host_checks=False + ) + removed = { + action.target + for action in second_plan.actions + if action.action == "remove" + } + + self.assertEqual( + {"redis", "worker", "scheduler", "test-mail"}, + removed, + ) + + def test_secret_change_is_planned_without_exposing_secret_values(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + paths = bundle_paths(Path(directory)) + paths.root.chmod(0o700) + spec = default_spec() + values = initial_secrets(spec) + write_env(paths.env, values) + first = build_plan(spec, paths, include_host_checks=False) + atomic_write( + paths.receipt, + canonical_json( + { + "spec_sha256": first.desired_spec_sha256, + "compose_sha256": first.desired_compose_sha256, + "environment_fingerprint": ( + first.desired_environment_fingerprint + ), + "services": list(render_compose(spec)["services"]), + } + ), + mode=0o600, + ) + rotated = dict(values) + rotated["REDIS_PASSWORD"] = "rotated-high-entropy-value" + write_env(paths.env, rotated) + + second = build_plan(spec, paths, include_host_checks=False) + plan_json = json.dumps(second.to_dict()) + + self.assertIn( + ("reconfigure", "environment"), + {(action.action, action.target) for action in second.actions}, + ) + self.assertNotIn(rotated["REDIS_PASSWORD"], plan_json) + self.assertEqual( + environment_fingerprint(rotated), + second.desired_environment_fingerprint, + ) + + def test_external_endpoint_preflight_reports_reachability(self) -> None: + connection = MagicMock() + connection.__enter__.return_value = connection + with patch.object( + deployment_planning.socket, + "create_connection", + return_value=connection, + ) as connect: + check = _endpoint_check( + "external.redis", + "External Redis", + "rediss://redis.example.test/0", + default_ports={"redis": 6379, "rediss": 6380}, + ) + + self.assertEqual("ok", check.level) + connect.assert_called_once_with( + ("redis.example.test", 6380), + timeout=1.5, + ) + + def test_cli_init_writes_private_idempotent_bundle(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + root = Path(directory) / "installation" + result, _stdout, _stderr = run_cli( + [ + "init", + "--non-interactive", + "--directory", + str(root), + "--redis", + "disabled", + "--mail", + "test-mail", + ] + ) + + self.assertEqual(0, result) + self.assertEqual(0o700, stat.S_IMODE(root.stat().st_mode)) + for name in ( + "installation.json", + "secrets.env", + "compose.json", + "plan.json", + ): + self.assertEqual( + 0o600, stat.S_IMODE((root / name).stat().st_mode) + ) + values = read_env(root / "secrets.env") + self.assertTrue(values["MASTER_KEY_B64"]) + self.assertNotIn( + values["POSTGRES_PASSWORD"], + (root / "compose.json").read_text(encoding="utf-8"), + ) + self.assertEqual( + 1, + run_cli( + [ + "init", + "--non-interactive", + "--directory", + str(root), + ] + )[0], + ) + + def test_cli_requires_external_url_when_switching_from_managed(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + root = Path(directory) / "installation" + self.assertEqual( + 0, + run_cli( + [ + "init", + "--non-interactive", + "--directory", + str(root), + ] + )[0], + ) + self.assertEqual( + 1, + run_cli( + [ + "configure", + "--directory", + str(root), + "--postgres", + "external", + ] + )[0], + ) + self.assertEqual( + "managed", + json.loads( + (root / "installation.json").read_text(encoding="utf-8") + )["components"]["postgres"]["mode"], + ) + self.assertEqual( + 1, + run_cli( + [ + "configure", + "--directory", + str(root), + "--installation-id", + "renamed-installation", + ] + )[0], + ) + + def test_deployer_builds_and_runs_as_one_zipapp(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + output = Path(directory) / "govoplan-deploy.pyz" + build = subprocess.run( + [ + sys.executable, + str(DEPLOYMENT_TOOLS / "build-deployer-zipapp.py"), + "--output", + str(output), + ], + cwd=META_ROOT, + check=False, + capture_output=True, + text=True, + ) + help_result = subprocess.run( + [sys.executable, str(output), "--help"], + cwd=META_ROOT, + check=False, + capture_output=True, + text=True, + ) + install_root = Path(directory) / "install" + init_result = subprocess.run( + [ + sys.executable, + str(output), + "init", + "--non-interactive", + "--directory", + str(install_root), + ], + cwd=META_ROOT, + check=False, + capture_output=True, + text=True, + ) + render_result = subprocess.run( + [ + sys.executable, + str(output), + "render", + "--directory", + str(install_root), + "--json", + ], + cwd=META_ROOT, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(0, build.returncode, build.stderr) + self.assertTrue(output.exists()) + self.assertEqual(0o755, stat.S_IMODE(output.stat().st_mode)) + self.assertEqual(0, help_result.returncode, help_result.stderr) + self.assertIn("govoplan-deploy", help_result.stdout) + self.assertEqual(0, init_result.returncode, init_result.stderr) + self.assertEqual(1, render_result.returncode, render_result.stderr) + self.assertTrue(json.loads(render_result.stdout)["blocked"]) + + def test_generated_environment_passes_core_startup_validation_when_available( + self, + ) -> None: + try: + from govoplan_core.core.install_config import ( + validate_runtime_configuration, + ) + except ModuleNotFoundError: + self.skipTest("govoplan-core is not installed in this test environment") + + for profile, public_url in ( + ("evaluation", "http://127.0.0.1:8080"), + ("self-hosted", "https://govoplan.example.test"), + ): + with self.subTest(profile=profile): + environment = initial_secrets( + default_spec(profile=profile, public_url=public_url) + ) + validation = validate_runtime_configuration(environment) + self.assertEqual( + (), + validation.errors, + validation.to_text(), + ) + + def test_apply_orders_dependencies_migration_and_runtime_before_receipt( + self, + ) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + root = Path(directory) / "installation" + self.assertEqual( + 0, + run_cli( + [ + "init", + "--non-interactive", + "--directory", + str(root), + "--api-image", + "local/govoplan-api:test", + "--web-image", + "local/govoplan-web:test", + ] + )[0], + ) + commands: list[list[str]] = [] + + def record_command(argv, *, cwd): + self.assertEqual(root, cwd) + commands.append(list(argv)) + + compose_version = subprocess.CompletedProcess( + args=["docker", "compose", "version"], + returncode=0, + stdout="2.30.0\n", + stderr="", + ) + with ( + patch.object( + deployment_cli.shutil, + "which", + return_value="/usr/bin/docker", + ), + patch.object( + deployment_planning.shutil, + "which", + return_value="/usr/bin/docker", + ), + patch.object( + deployment_planning, + "_run_command", + return_value=compose_version, + ), + patch.object(deployment_cli, "_run", side_effect=record_command), + patch.object(deployment_cli, "_wait_for_health") as wait_for_health, + ): + result, _stdout, _stderr = run_cli( + [ + "apply", + "--directory", + str(root), + "--skip-pull", + "--allow-unverified-images", + ] + ) + + self.assertEqual(0, result) + migrate_index = next( + index + for index, command in enumerate(commands) + if command[-3:] == ["run", "--rm", "migrate"] + ) + runtime_index = next( + index + for index, command in enumerate(commands) + if "--remove-orphans" in command + ) + self.assertLess(migrate_index, runtime_index) + self.assertIn("postgres", commands[0]) + self.assertIn("redis", commands[0]) + wait_for_health.assert_called_once_with( + "http://127.0.0.1:8080/health", + timeout_seconds=120.0, + ) + receipt = json.loads( + (root / "receipt.json").read_text(encoding="utf-8") + ) + self.assertEqual("govoplan-local", receipt["installation_id"]) + self.assertEqual( + {"address": "127.0.0.1", "port": 8080}, + receipt["listen"], + ) + self.assertNotIn("installer", receipt["services"]) + + def test_installation_root_symlink_is_rejected(self) -> None: + if not hasattr(Path, "symlink_to"): + self.skipTest("symbolic links are unavailable") + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + base = Path(directory) + target = base / "real" + target.mkdir() + link = base / "linked" + link.symlink_to(target, target_is_directory=True) + + with self.assertRaisesRegex(ValueError, "symbolic link"): + bundle_paths(link) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/checks/check-focused.sh b/tools/checks/check-focused.sh index c540080..499ffdc 100644 --- a/tools/checks/check-focused.sh +++ b/tools/checks/check-focused.sh @@ -30,6 +30,10 @@ GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash "$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" +cd "$META_ROOT" +"$PYTHON" -m unittest tests.test_deployment_installer +cd "$ROOT" + "$PYTHON" - <<'PY' import ast import pathlib diff --git a/tools/deployment/__main__.py b/tools/deployment/__main__.py new file mode 100644 index 0000000..6ebed0c --- /dev/null +++ b/tools/deployment/__main__.py @@ -0,0 +1,6 @@ +"""Executable entry point for the single-file GovOPlaN deployer.""" + +from govoplan_deploy.cli import main + + +raise SystemExit(main()) diff --git a/tools/deployment/build-deployer-zipapp.py b/tools/deployment/build-deployer-zipapp.py new file mode 100644 index 0000000..01370f2 --- /dev/null +++ b/tools/deployment/build-deployer-zipapp.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Build the dependency-free GovOPlaN deployer as one executable zipapp.""" + +from __future__ import annotations + +import argparse +from hashlib import sha256 +from pathlib import Path +import zipapp + + +ROOT = Path(__file__).resolve().parent +DEFAULT_OUTPUT = ROOT.parent.parent / "runtime" / "deployment" / "govoplan-deploy.pyz" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + args = parser.parse_args(argv) + + output = args.output.expanduser().resolve() + if output == ROOT or ROOT in output.parents: + raise SystemExit("output must be outside the deployment source directory") + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = output.with_name(f".{output.name}.tmp") + if temporary.exists(): + temporary.unlink() + zipapp.create_archive( + ROOT, + target=temporary, + interpreter="/usr/bin/env python3", + compressed=True, + filter=_include_source, + ) + temporary.chmod(0o755) + temporary.replace(output) + digest = sha256(output.read_bytes()).hexdigest() + print(f"{digest} {output}") + return 0 + + +def _include_source(path: Path) -> bool: + return ( + "__pycache__" not in path.parts + and path.suffix not in {".pyc", ".pyo"} + and path.name != "build-deployer-zipapp.py" + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deployment/govoplan-deploy.py b/tools/deployment/govoplan-deploy.py new file mode 100644 index 0000000..cffc04b --- /dev/null +++ b/tools/deployment/govoplan-deploy.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""GovOPlaN deployment entry point.""" + +from __future__ import annotations + +from pathlib import Path +import sys + + +TOOLS_ROOT = Path(__file__).resolve().parent +if str(TOOLS_ROOT) not in sys.path: + sys.path.insert(0, str(TOOLS_ROOT)) + +from govoplan_deploy.cli import main # noqa: E402 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/deployment/govoplan_deploy/__init__.py b/tools/deployment/govoplan_deploy/__init__.py new file mode 100644 index 0000000..5211521 --- /dev/null +++ b/tools/deployment/govoplan_deploy/__init__.py @@ -0,0 +1,19 @@ +"""Declarative GovOPlaN host deployment tooling.""" + +from .model import ( + BASE_MODULES, + FULL_MODULES, + InstallationSpec, + SpecError, + default_spec, + load_spec, +) + +__all__ = [ + "BASE_MODULES", + "FULL_MODULES", + "InstallationSpec", + "SpecError", + "default_spec", + "load_spec", +] diff --git a/tools/deployment/govoplan_deploy/bundle.py b/tools/deployment/govoplan_deploy/bundle.py new file mode 100644 index 0000000..60f5d72 --- /dev/null +++ b/tools/deployment/govoplan_deploy/bundle.py @@ -0,0 +1,572 @@ +"""Secret handling and deterministic Compose bundle rendering.""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from hashlib import sha256 +import hmac +import json +import os +from pathlib import Path +import secrets +import stat +from typing import Mapping +from urllib.parse import quote, urlsplit + +from .model import ENV_NAME_PATTERN, InstallationSpec + + +SPEC_FILENAME = "installation.json" +ENV_FILENAME = "secrets.env" +COMPOSE_FILENAME = "compose.json" +PLAN_FILENAME = "plan.json" +RECEIPT_FILENAME = "receipt.json" +LOCK_FILENAME = ".deployment.lock" +RUNTIME_ENV_KEYS = ( + "APP_ENV", + "GOVOPLAN_INSTALL_PROFILE", + "MASTER_KEY_B64", + "DATABASE_URL", + "GOVOPLAN_DATABASE_URL_PGTOOLS", + "ENABLED_MODULES", + "CELERY_ENABLED", + "CELERY_QUEUES", + "REDIS_URL", + "CORS_ORIGINS", + "GOVOPLAN_TRUSTED_HOSTS", + "FORWARDED_ALLOW_IPS", + "AUTH_COOKIE_SECURE", + "AUTH_COOKIE_SAMESITE", + "GOVOPLAN_HTTP_HSTS_SECONDS", + "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", + "GOVOPLAN_MIGRATION_TRACK", + "DEV_AUTO_MIGRATE_ENABLED", + "DEV_BOOTSTRAP_ENABLED", + "GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE", + "GOVOPLAN_DEPLOYMENT_SPEC_PATH", + "FILE_STORAGE_BACKEND", + "FILE_STORAGE_LOCAL_ROOT", + "FILE_STORAGE_S3_ENDPOINT_URL", + "FILE_STORAGE_S3_REGION", + "FILE_STORAGE_S3_ACCESS_KEY_ID", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY", + "FILE_STORAGE_S3_BUCKET", +) + + +@dataclass(frozen=True, slots=True) +class BundlePaths: + root: Path + spec: Path + env: Path + compose: Path + plan: Path + receipt: Path + lock: Path + + +def bundle_paths(root: Path) -> BundlePaths: + expanded = root.expanduser() + if expanded.is_symlink(): + raise ValueError( + f"installation root must not be a symbolic link: {expanded}" + ) + resolved = expanded.resolve() + return BundlePaths( + root=resolved, + spec=resolved / SPEC_FILENAME, + env=resolved / ENV_FILENAME, + compose=resolved / COMPOSE_FILENAME, + plan=resolved / PLAN_FILENAME, + receipt=resolved / RECEIPT_FILENAME, + lock=resolved / LOCK_FILENAME, + ) + + +def ensure_private_directory(path: Path) -> None: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"installation root must be a real directory: {path}") + current = stat.S_IMODE(path.stat().st_mode) + if current & 0o077: + path.chmod(0o700) + + +def initial_secrets( + spec: InstallationSpec, + *, + supplied: Mapping[str, str] | None = None, +) -> dict[str, str]: + values = dict(supplied or {}) + values.setdefault( + "MASTER_KEY_B64", + base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"), + ) + values.setdefault("POSTGRES_DB", "govoplan") + values.setdefault("POSTGRES_USER", "govoplan") + values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) + values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) + return reconcile_runtime_environment(spec, values) + + +def reconcile_runtime_environment( + spec: InstallationSpec, + current: Mapping[str, str], +) -> dict[str, str]: + values = dict(current) + postgres = spec.components.postgres + if postgres.mode == "managed": + database = values.setdefault("POSTGRES_DB", "govoplan") + username = values.setdefault("POSTGRES_USER", "govoplan") + password = values.setdefault("POSTGRES_PASSWORD", secrets.token_urlsafe(36)) + encoded_user = quote(username, safe="") + encoded_password = quote(password, safe="") + encoded_database = quote(database, safe="") + values["DATABASE_URL"] = ( + f"postgresql+psycopg://{encoded_user}:{encoded_password}" + f"@postgres:5432/{encoded_database}" + ) + values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = ( + f"postgresql://{encoded_user}:{encoded_password}" + f"@postgres:5432/{encoded_database}" + ) + elif not values.get(postgres.url_env): + raise ValueError( + f"external PostgreSQL requires {postgres.url_env} in {ENV_FILENAME}" + ) + else: + _validate_service_url( + values[postgres.url_env], + schemes={"postgresql", "postgresql+psycopg"}, + label="external PostgreSQL URL", + ) + + redis = spec.components.redis + if redis.mode == "managed": + password = values.setdefault("REDIS_PASSWORD", secrets.token_urlsafe(36)) + values["REDIS_URL"] = f"redis://:{quote(password, safe='')}@redis:6379/0" + elif redis.mode == "external": + if not values.get(redis.url_env): + raise ValueError( + f"external Redis requires {redis.url_env} in {ENV_FILENAME}" + ) + _validate_service_url( + values[redis.url_env], + schemes={"redis", "rediss"}, + label="external Redis URL", + ) + else: + values["REDIS_URL"] = "" + + public = urlsplit(spec.public_url) + values.update( + { + "APP_ENV": "production" if spec.profile == "self-hosted" else "staging", + "GOVOPLAN_INSTALL_PROFILE": spec.profile, + "ENABLED_MODULES": ",".join(spec.enabled_modules), + "CELERY_ENABLED": "true" if redis.mode != "disabled" else "false", + "CELERY_QUEUES": ( + "send_email,append_sent,notifications,calendar,dataflow,events,default" + ), + "CORS_ORIGINS": spec.public_url, + "GOVOPLAN_TRUSTED_HOSTS": public.hostname or "", + "FORWARDED_ALLOW_IPS": spec.network_subnet, + "AUTH_COOKIE_SECURE": "true" if public.scheme == "https" else "false", + "AUTH_COOKIE_SAMESITE": "lax", + "GOVOPLAN_HTTP_HSTS_SECONDS": ( + "31536000" if public.scheme == "https" else "0" + ), + "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false", + "GOVOPLAN_MIGRATION_TRACK": "release", + "DEV_AUTO_MIGRATE_ENABLED": "false", + "DEV_BOOTSTRAP_ENABLED": "false", + "GOVOPLAN_DEPLOYMENT_SPEC_PATH": "/etc/govoplan/deployment/installation.json", + } + ) + if redis.mode == "disabled": + values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "true" + else: + values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false" + + storage = spec.components.storage + values["FILE_STORAGE_BACKEND"] = storage.mode + if storage.mode == "local": + values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files" + else: + required = ( + "FILE_STORAGE_S3_ENDPOINT_URL", + "FILE_STORAGE_S3_REGION", + "FILE_STORAGE_S3_ACCESS_KEY_ID", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY", + "FILE_STORAGE_S3_BUCKET", + ) + missing = [name for name in required if not values.get(name)] + if missing: + raise ValueError( + "S3 storage requires values in secrets.env: " + ", ".join(missing) + ) + endpoint = _validate_service_url( + values["FILE_STORAGE_S3_ENDPOINT_URL"], + schemes={"http", "https"}, + label="S3 endpoint URL", + ) + if spec.profile == "self-hosted" and endpoint.scheme != "https": + raise ValueError("self-hosted S3 endpoint URL must use HTTPS") + return dict(sorted(values.items())) + + +def render_compose(spec: InstallationSpec) -> dict[str, object]: + runtime_env = { + "environment": _environment_references(RUNTIME_ENV_KEYS), + } + deployment_mount = ( + f"./{SPEC_FILENAME}:/etc/govoplan/deployment/installation.json:ro" + ) + data_mounts = [deployment_mount] + if spec.components.storage.mode == "local": + data_mounts.append("files-data:/var/lib/govoplan/files") + + dependency_conditions: dict[str, dict[str, str]] = {} + services: dict[str, object] = {} + if spec.components.postgres.mode == "managed": + services["postgres"] = { + "image": spec.components.postgres.image, + "restart": "unless-stopped", + "environment": _environment_references( + ("POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"), + required=True, + ), + "healthcheck": { + "test": [ + "CMD-SHELL", + 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"', + ], + "interval": "5s", + "timeout": "3s", + "retries": 30, + }, + "volumes": ["postgres-data:/var/lib/postgresql/data"], + "networks": ["internal"], + } + dependency_conditions["postgres"] = {"condition": "service_healthy"} + + if spec.components.redis.mode == "managed": + services["redis"] = { + "image": spec.components.redis.image, + "restart": "unless-stopped", + "environment": _environment_references( + ("REDIS_PASSWORD",), + required=True, + ), + "command": [ + "sh", + "-ec", + 'exec redis-server --appendonly yes --requirepass "$$REDIS_PASSWORD"', + ], + "healthcheck": { + "test": [ + "CMD-SHELL", + 'redis-cli -a "$${REDIS_PASSWORD}" --no-auth-warning ping', + ], + "interval": "5s", + "timeout": "3s", + "retries": 30, + }, + "volumes": ["redis-data:/data"], + "networks": ["internal"], + } + dependency_conditions["redis"] = {"condition": "service_healthy"} + + if spec.components.mail.mode == "test-mail": + services["test-mail"] = { + "image": spec.components.mail.image, + "restart": "unless-stopped", + "environment": { + "GREENMAIL_OPTS": ( + "-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap " + "-Dgreenmail.hostname=0.0.0.0" + ) + }, + "networks": ["internal"], + } + + common_runtime: dict[str, object] = { + **runtime_env, + "restart": "unless-stopped", + "volumes": data_mounts, + "networks": ["internal"], + } + if dependency_conditions: + common_runtime["depends_on"] = dependency_conditions + + services["migrate"] = { + **runtime_env, + "image": spec.release.api_image, + "command": ["python", "-m", "govoplan_core.commands.init_db"], + "restart": "no", + "volumes": data_mounts, + "networks": ["internal"], + **({"depends_on": dependency_conditions} if dependency_conditions else {}), + } + services["api"] = { + **common_runtime, + "image": spec.release.api_image, + "command": [ + "python", + "-m", + "uvicorn", + "govoplan_core.server.app:app", + "--host", + "0.0.0.0", + "--port", + "8000", + "--proxy-headers", + ], + "healthcheck": { + "test": [ + "CMD", + "python", + "-c", + ( + "import urllib.request;" + "urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)" + ), + ], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "20s", + }, + } + services["web"] = { + "image": spec.release.web_image, + "restart": "unless-stopped", + "environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"}, + "depends_on": {"api": {"condition": "service_healthy"}}, + "ports": [_published_port(spec.listen.address, spec.listen.port, 8080)], + "networks": ["internal"], + } + if spec.components.redis.mode != "disabled": + services["worker"] = { + **common_runtime, + "image": spec.release.api_image, + "command": [ + "python", + "-m", + "celery", + "-A", + "govoplan_core.celery_app:celery", + "worker", + "--queues", + ( + "send_email,append_sent,notifications,calendar," + "dataflow,events,default" + ), + "--loglevel", + "INFO", + ], + } + services["scheduler"] = { + **common_runtime, + "image": spec.release.api_image, + "command": [ + "python", + "-m", + "celery", + "-A", + "govoplan_core.celery_app:celery", + "beat", + "--loglevel", + "INFO", + ], + } + volumes: dict[str, object] = {} + if spec.components.postgres.mode == "managed": + volumes["postgres-data"] = {} + if spec.components.redis.mode == "managed": + volumes["redis-data"] = {} + if spec.components.storage.mode == "local": + volumes["files-data"] = {} + + return { + "name": spec.installation_id, + "services": services, + "volumes": volumes, + "networks": { + "internal": { + "driver": "bridge", + "ipam": {"config": [{"subnet": spec.network_subnet}]}, + } + }, + } + + +def service_names(spec: InstallationSpec) -> tuple[str, ...]: + return tuple(render_compose(spec)["services"].keys()) + + +def canonical_json(value: object) -> bytes: + return ( + json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + + "\n" + ).encode("utf-8") + + +def digest_json(value: object) -> str: + return sha256(canonical_json(value)).hexdigest() + + +def environment_fingerprint(values: Mapping[str, str]) -> str: + key = values.get("MASTER_KEY_B64", "").encode("utf-8") + if not key: + return "" + payload = canonical_json(dict(sorted(values.items()))) + return hmac.new(key, payload, sha256).hexdigest() + + +def read_env(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + values: dict[str, str] = {} + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, separator, raw_value = line.partition("=") + if not separator or not ENV_NAME_PATTERN.fullmatch(key): + raise ValueError(f"invalid environment line {line_number} in {path}") + if key in values: + raise ValueError( + f"duplicate environment key {key!r} on line {line_number}" + ) + value = raw_value + if value.startswith('"'): + try: + decoded = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError( + f"invalid quoted environment value on line {line_number}" + ) from exc + if not isinstance(decoded, str): + raise ValueError( + f"environment value on line {line_number} must be a string" + ) + value = decoded + elif value.startswith("'"): + value = _single_quoted_env_value(value, line_number=line_number) + values[key] = value + return values + + +def write_env(path: Path, values: Mapping[str, str]) -> None: + invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key)) + if invalid: + raise ValueError( + "invalid environment variable names: " + ", ".join(invalid) + ) + lines = [ + "# Generated by govoplan-deploy. Keep this file private.", + *[f"{key}={_env_value(value)}" for key, value in sorted(values.items())], + "", + ] + atomic_write(path, "\n".join(lines).encode("utf-8"), mode=0o600) + + +def atomic_write(path: Path, payload: bytes, *, mode: int) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_name( + f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" + ) + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + mode, + ) + try: + with os.fdopen(descriptor, "wb", closefd=True) as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + path.chmod(mode) + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if temporary.exists(): + temporary.unlink() + + +def _env_value(value: str) -> str: + if "\x00" in value or "\n" in value or "\r" in value: + raise ValueError("environment values must not contain NUL or line breaks") + if value and all( + character.isalnum() or character in "_./:@%+,-" + for character in value + ): + return value + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def _validate_service_url( + value: str, + *, + schemes: set[str], + label: str, +): + parsed = urlsplit(value) + try: + parsed.port + except ValueError as exc: + raise ValueError(f"{label} contains an invalid port") from exc + if parsed.scheme not in schemes or not parsed.hostname: + raise ValueError( + f"{label} must use one of {', '.join(sorted(schemes))} and include a host" + ) + if parsed.fragment: + raise ValueError(f"{label} must not contain a fragment") + return parsed + + +def _single_quoted_env_value(value: str, *, line_number: int) -> str: + if len(value) < 2 or not value.endswith("'"): + raise ValueError( + f"unterminated single-quoted environment value on line {line_number}" + ) + body = value[1:-1] + output: list[str] = [] + index = 0 + while index < len(body): + character = body[index] + if character != "\\": + output.append(character) + index += 1 + continue + index += 1 + if index >= len(body) or body[index] not in {"\\", "'"}: + raise ValueError( + f"invalid single-quoted escape on line {line_number}" + ) + output.append(body[index]) + index += 1 + return "".join(output) + + +def _published_port(address: str, host_port: int, container_port: int) -> str: + host = f"[{address}]" if ":" in address else address + return f"{host}:{host_port}:{container_port}" + + +def _environment_references( + keys: tuple[str, ...], + *, + required: bool = False, +) -> dict[str, str]: + suffix = ":?required by GovOPlaN deployment" if required else ":-" + return {key: f"${{{key}{suffix}}}" for key in keys} diff --git a/tools/deployment/govoplan_deploy/cli.py b/tools/deployment/govoplan_deploy/cli.py new file mode 100644 index 0000000..9af05d4 --- /dev/null +++ b/tools/deployment/govoplan_deploy/cli.py @@ -0,0 +1,692 @@ +"""Command-line interface for GovOPlaN deployment reconciliation.""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from dataclasses import replace +from datetime import UTC, datetime +import fcntl +import getpass +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +from typing import Iterator, Mapping, Sequence +from urllib.error import URLError +from urllib.request import urlopen + +from .bundle import ( + atomic_write, + bundle_paths, + canonical_json, + digest_json, + environment_fingerprint, + ensure_private_directory, + initial_secrets, + read_env, + reconcile_runtime_environment, + render_compose, + service_names, + write_env, +) +from .model import ( + ComponentConfig, + InstallationSpec, + ListenConfig, + SpecError, + default_spec, + load_spec, + parse_spec, +) +from .planning import DeploymentPlan, build_plan + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="govoplan-deploy", + description=( + "Create, validate, and reconcile a declarative GovOPlaN Compose deployment." + ), + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + init = subparsers.add_parser( + "init", help="Create a new private installation bundle." + ) + _directory_argument(init) + _configuration_arguments(init) + init.add_argument( + "--non-interactive", + action="store_true", + help="Use flags/defaults without prompting.", + ) + + configure = subparsers.add_parser( + "configure", + help="Change installation choices while preserving generated secrets.", + ) + _directory_argument(configure) + _configuration_arguments(configure, defaults=False) + + render = subparsers.add_parser( + "render", help="Regenerate Compose and the deployment plan without applying." + ) + _directory_argument(render) + render.add_argument("--json", action="store_true", help="Print the plan as JSON.") + + doctor = subparsers.add_parser( + "doctor", help="Run specification, secret, host, and provenance checks." + ) + _directory_argument(doctor) + doctor.add_argument("--json", action="store_true", help="Print the plan as JSON.") + + apply_parser = subparsers.add_parser( + "apply", help="Apply an allowed plan with Docker Compose." + ) + _directory_argument(apply_parser) + apply_parser.add_argument( + "--allow-unverified-images", + action="store_true", + help="Allow mutable/unverified images for an evaluation profile only.", + ) + apply_parser.add_argument( + "--skip-pull", + action="store_true", + help="Do not pull images before reconciliation.", + ) + apply_parser.add_argument( + "--health-timeout-seconds", + type=float, + default=120.0, + help="Maximum time to wait for the public health endpoint.", + ) + + status = subparsers.add_parser( + "status", help="Show desired state and current Compose process state." + ) + _directory_argument(status) + status.add_argument("--json", action="store_true", help="Print JSON.") + return parser + + +def _directory_argument(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--directory", + type=Path, + default=Path.home() / ".local" / "share" / "govoplan" / "installations" / "default", + help="Private installation state directory.", + ) + + +def _configuration_arguments( + parser: argparse.ArgumentParser, *, defaults: bool = True +) -> None: + default = (lambda value: value) if defaults else (lambda _value: None) + parser.add_argument("--installation-id", default=default("govoplan-local")) + parser.add_argument( + "--profile", + choices=("evaluation", "self-hosted"), + default=default("evaluation"), + ) + parser.add_argument("--public-url", default=default("http://127.0.0.1:8080")) + parser.add_argument("--listen-address", default=default("127.0.0.1")) + parser.add_argument("--listen-port", type=int, default=default(8080)) + parser.add_argument( + "--postgres", + choices=("managed", "external"), + default=default("managed"), + ) + parser.add_argument("--database-url", help="External PostgreSQL URL; stored privately.") + parser.add_argument( + "--redis", + choices=("managed", "external", "disabled"), + default=default("managed"), + ) + parser.add_argument("--redis-url", help="External Redis URL; stored privately.") + parser.add_argument( + "--mail", + choices=("disabled", "external-relay", "test-mail"), + default=default("disabled"), + ) + parser.add_argument( + "--storage", + choices=("local", "s3"), + default=default("local"), + ) + parser.add_argument("--s3-endpoint-url", help="S3-compatible endpoint URL.") + parser.add_argument("--s3-region", help="S3 region.") + parser.add_argument("--s3-access-key-id", help="S3 access key id; stored privately.") + parser.add_argument( + "--s3-secret-access-key", + help="S3 secret access key; stored privately.", + ) + parser.add_argument("--s3-bucket", help="S3 bucket for managed files.") + parser.add_argument( + "--module-set", + choices=("core", "base", "full"), + default=default("base"), + ) + parser.add_argument("--api-image", default=default("govoplan-api:unpublished")) + parser.add_argument("--web-image", default=default("govoplan-web:unpublished")) + parser.add_argument("--release-version", default=default("unpublished")) + parser.add_argument("--release-channel", default=default("stable")) + parser.add_argument("--manifest-url", default=default("")) + parser.add_argument("--manifest-sha256", default=default("")) + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "init": + return _init(args) + if args.command == "configure": + return _configure(args) + if args.command in {"render", "doctor"}: + return _render_or_doctor(args) + if args.command == "apply": + return _apply(args) + if args.command == "status": + return _status(args) + except (SpecError, ValueError, OSError, subprocess.SubprocessError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + raise RuntimeError(f"unsupported command: {args.command}") + + +def _init(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + ensure_private_directory(paths.root) + if paths.spec.exists(): + raise ValueError( + f"{paths.spec} already exists; use configure for an existing installation" + ) + if not args.non_interactive and sys.stdin.isatty(): + _prompt_configuration(args) + spec = default_spec( + installation_id=args.installation_id, + profile=args.profile, + public_url=args.public_url, + listen_address=args.listen_address, + listen_port=args.listen_port, + postgres_mode=args.postgres, + redis_mode=args.redis, + mail_mode=args.mail, + storage_mode=args.storage, + module_set=args.module_set, + api_image=args.api_image, + web_image=args.web_image, + manifest_url=args.manifest_url, + manifest_sha256=args.manifest_sha256, + version=args.release_version, + channel=args.release_channel, + ) + supplied = _supplied_secret_values(args) + secrets = initial_secrets(spec, supplied=supplied) + _write_bundle(spec, paths, secrets) + plan = build_plan(spec, paths, include_host_checks=False) + _write_plan(paths.plan, plan) + print(f"Created GovOPlaN installation bundle at {paths.root}") + print(f"Edit choices with: govoplan-deploy configure --directory {paths.root}") + print(f"Check readiness with: govoplan-deploy doctor --directory {paths.root}") + if any(check.level == "error" for check in plan.checks): + print("The bundle is not apply-ready yet; run doctor for the blocking checks.") + return 0 + + +def _configure(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + ensure_private_directory(paths.root) + current = load_spec(paths.spec) + if args.installation_id and args.installation_id != current.installation_id: + raise ValueError( + "installation_id is immutable; create a separate installation " + "instead of renaming a Compose project" + ) + spec = _updated_spec(current, args) + if ( + current.components.postgres.mode == "managed" + and spec.components.postgres.mode == "external" + and not args.database_url + ): + raise ValueError( + "switching PostgreSQL from managed to external requires --database-url" + ) + if ( + current.components.redis.mode != "external" + and spec.components.redis.mode == "external" + and not args.redis_url + ): + raise ValueError( + "switching Redis to external requires --redis-url" + ) + secrets = read_env(paths.env) + secrets.update(_supplied_secret_values(args)) + secrets = reconcile_runtime_environment(spec, secrets) + _write_bundle(spec, paths, secrets) + plan = build_plan(spec, paths, include_host_checks=False) + _write_plan(paths.plan, plan) + print(f"Updated desired state at {paths.root}") + _print_plan(plan) + return 0 + + +def _render_or_doctor(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + spec = load_spec(paths.spec) + secrets = reconcile_runtime_environment(spec, read_env(paths.env)) + _write_bundle(spec, paths, secrets) + plan = build_plan( + spec, + paths, + include_host_checks=args.command == "doctor", + ) + _write_plan(paths.plan, plan) + if args.json: + print(json.dumps(plan.to_dict(), indent=2, sort_keys=True)) + else: + _print_plan(plan) + return 1 if plan.blocked else 0 + + +def _apply(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + spec = load_spec(paths.spec) + if args.allow_unverified_images and spec.profile != "evaluation": + raise ValueError( + "--allow-unverified-images is restricted to evaluation installations" + ) + ensure_private_directory(paths.root) + with _deployment_lock(paths.lock): + secrets = reconcile_runtime_environment(spec, read_env(paths.env)) + _write_bundle(spec, paths, secrets) + plan = build_plan(spec, paths, include_host_checks=True) + _write_plan(paths.plan, plan) + effective_errors = [ + check + for check in plan.checks + if check.level == "error" + and not ( + args.allow_unverified_images + and check.id.startswith( + ( + "release.api_image.", + "release.web_image.", + "components.postgres.image.", + "components.redis.image.", + "components.mail.image.", + "release.manifest", + "modules.image_composition", + ) + ) + ) + ] + if effective_errors: + _print_plan(plan) + raise ValueError("deployment plan is blocked; resolve doctor errors first") + docker = shutil.which("docker") + if docker is None: + raise ValueError("Docker CLI is required for apply") + compose = [ + docker, + "compose", + "--env-file", + str(paths.env), + "--project-name", + spec.installation_id, + "--file", + str(paths.compose), + ] + if not args.skip_pull: + _run([*compose, "pull"], cwd=paths.root) + dependencies = [ + name + for name in ("postgres", "redis", "test-mail") + if name in service_names(spec) + ] + if dependencies: + _run([*compose, "up", "--detach", *dependencies], cwd=paths.root) + _run([*compose, "run", "--rm", "migrate"], cwd=paths.root) + runtime_services = [ + name + for name in ("api", "web", "worker", "scheduler") + if name in service_names(spec) + ] + _run( + [*compose, "up", "--detach", "--remove-orphans", *runtime_services], + cwd=paths.root, + ) + _wait_for_health( + f"{spec.public_url}/health", + timeout_seconds=args.health_timeout_seconds, + ) + receipt = { + "schema_version": 1, + "installation_id": spec.installation_id, + "applied_at": _now(), + "spec_sha256": digest_json(spec.to_dict()), + "compose_sha256": digest_json(render_compose(spec)), + "environment_fingerprint": environment_fingerprint(secrets), + "release": { + "channel": spec.release.channel, + "version": spec.release.version, + "manifest_sha256": spec.release.manifest_sha256, + "api_image": spec.release.api_image, + "web_image": spec.release.web_image, + }, + "services": list(service_names(spec)), + "listen": { + "address": spec.listen.address, + "port": spec.listen.port, + }, + "management": { + "mode": "govoplan-deploy", + "agent": "cli", + "web_updates": False, + }, + } + atomic_write(paths.receipt, canonical_json(receipt), mode=0o600) + print(f"GovOPlaN is ready at {spec.public_url}") + return 0 + + +def _status(args: argparse.Namespace) -> int: + paths = bundle_paths(args.directory) + spec = load_spec(paths.spec) + plan = build_plan(spec, paths, include_host_checks=False) + docker = shutil.which("docker") + processes: object = [] + process_error = "" + if docker: + result = subprocess.run( + [ + docker, + "compose", + "--env-file", + str(paths.env), + "--project-name", + spec.installation_id, + "--file", + str(paths.compose), + "ps", + "--format", + "json", + ], + cwd=paths.root, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + if result.returncode == 0: + processes = _parse_compose_ps(result.stdout) + else: + process_error = result.stderr.strip() or result.stdout.strip() + else: + process_error = "Docker CLI is unavailable." + payload = { + "installation_id": spec.installation_id, + "public_url": spec.public_url, + "plan": plan.to_dict(), + "processes": processes, + "process_error": process_error, + } + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + _print_plan(plan) + if process_error: + print(f"Runtime: {process_error}") + elif isinstance(processes, list): + print(f"Runtime: {len(processes)} Compose process(es) reported.") + return 0 + + +def _updated_spec( + current: InstallationSpec, args: argparse.Namespace +) -> InstallationSpec: + module_set = getattr(args, "module_set", None) + if module_set: + modules = default_spec(module_set=module_set).enabled_modules + else: + modules = current.enabled_modules + release = replace( + current.release, + channel=args.release_channel or current.release.channel, + version=args.release_version or current.release.version, + manifest_url=( + args.manifest_url + if args.manifest_url is not None + else current.release.manifest_url + ), + manifest_sha256=( + args.manifest_sha256 + if args.manifest_sha256 is not None + else current.release.manifest_sha256 + ), + api_image=args.api_image or current.release.api_image, + web_image=args.web_image or current.release.web_image, + ) + components = ComponentConfig( + postgres=replace( + current.components.postgres, + mode=args.postgres or current.components.postgres.mode, + ), + redis=replace( + current.components.redis, + mode=args.redis or current.components.redis.mode, + ), + mail=replace( + current.components.mail, + mode=args.mail or current.components.mail.mode, + ), + storage=replace( + current.components.storage, + mode=args.storage or current.components.storage.mode, + ), + ) + value = replace( + current, + installation_id=args.installation_id or current.installation_id, + profile=args.profile or current.profile, + public_url=args.public_url or current.public_url, + listen=ListenConfig( + address=args.listen_address or current.listen.address, + port=args.listen_port or current.listen.port, + ), + release=release, + components=components, + enabled_modules=modules, + ) + return parse_spec(value.to_dict()) + + +def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]: + values: dict[str, str] = {} + if getattr(args, "database_url", None): + values["DATABASE_URL"] = args.database_url + values["GOVOPLAN_DATABASE_URL_PGTOOLS"] = _pgtools_url(args.database_url) + if getattr(args, "redis_url", None): + values["REDIS_URL"] = args.redis_url + s3_values = { + "FILE_STORAGE_S3_ENDPOINT_URL": getattr(args, "s3_endpoint_url", None), + "FILE_STORAGE_S3_REGION": getattr(args, "s3_region", None), + "FILE_STORAGE_S3_ACCESS_KEY_ID": getattr( + args, "s3_access_key_id", None + ), + "FILE_STORAGE_S3_SECRET_ACCESS_KEY": getattr( + args, "s3_secret_access_key", None + ), + "FILE_STORAGE_S3_BUCKET": getattr(args, "s3_bucket", None), + } + values.update({key: value for key, value in s3_values.items() if value}) + return values + + +def _pgtools_url(database_url: str) -> str: + if database_url.startswith("postgresql+psycopg://"): + return "postgresql://" + database_url.removeprefix( + "postgresql+psycopg://" + ) + return database_url + + +def _write_bundle( + spec: InstallationSpec, + paths, + secrets: Mapping[str, str], +) -> None: + ensure_private_directory(paths.root) + atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600) + write_env(paths.env, secrets) + atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600) + + +def _write_plan(path: Path, plan: DeploymentPlan) -> None: + atomic_write(path, canonical_json(plan.to_dict()), mode=0o600) + + +def _print_plan(plan: DeploymentPlan) -> None: + state = "BLOCKED" if plan.blocked else "READY" + print(f"GovOPlaN deployment plan: {state}") + for action in plan.actions: + print(f" {action.action:12} {action.target}: {action.detail}") + for check in plan.checks: + prefix = {"ok": "OK", "warning": "WARN", "error": "ERROR"}.get( + check.level, check.level.upper() + ) + print(f" [{prefix}] {check.message}") + if check.action and check.level != "ok": + print(f" {check.action}") + + +def _prompt_configuration(args: argparse.Namespace) -> None: + args.profile = _prompt_choice( + "Installation profile", args.profile, ("evaluation", "self-hosted") + ) + if args.profile == "self-hosted" and args.public_url.startswith("http://"): + args.public_url = "https://govoplan.example.org" + args.public_url = _prompt("Public URL", args.public_url) + args.postgres = _prompt_choice( + "PostgreSQL", args.postgres, ("managed", "external") + ) + if args.postgres == "external" and not args.database_url: + args.database_url = getpass.getpass( + "External PostgreSQL URL (input hidden): " + ).strip() + redis_choices = ( + ("managed", "external") + if args.profile == "self-hosted" + else ("managed", "external", "disabled") + ) + args.redis = _prompt_choice( + "Redis", args.redis, redis_choices + ) + if args.redis == "external" and not args.redis_url: + args.redis_url = getpass.getpass( + "External Redis URL (input hidden): " + ).strip() + mail_choices = ( + ("disabled", "external-relay") + if args.profile == "self-hosted" + else ("disabled", "external-relay", "test-mail") + ) + args.mail = _prompt_choice( + "Mail integration", + args.mail, + mail_choices, + ) + args.storage = _prompt_choice("File storage", args.storage, ("local", "s3")) + if args.storage == "s3": + args.s3_endpoint_url = args.s3_endpoint_url or _prompt( + "S3 endpoint URL", "https://s3.example.org" + ) + args.s3_region = args.s3_region or _prompt("S3 region", "eu-central-1") + args.s3_access_key_id = args.s3_access_key_id or _prompt( + "S3 access key id", "" + ) + args.s3_secret_access_key = ( + args.s3_secret_access_key + or getpass.getpass("S3 secret access key (input hidden): ").strip() + ) + args.s3_bucket = args.s3_bucket or _prompt("S3 bucket", "govoplan-files") + args.module_set = _prompt_choice( + "Initial module set", args.module_set, ("core", "base", "full") + ) + + +def _prompt(label: str, default: str) -> str: + value = input(f"{label} [{default}]: ").strip() + return value or default + + +def _prompt_choice(label: str, default: str, choices: tuple[str, ...]) -> str: + while True: + value = _prompt(f"{label} ({'/'.join(choices)})", default) + if value in choices: + return value + print(f"Choose one of: {', '.join(choices)}") + + +@contextmanager +def _deployment_lock(path: Path) -> Iterator[None]: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise ValueError("another deployment operation owns the installation lock") from exc + os.ftruncate(descriptor, 0) + os.write(descriptor, f"{os.getpid()}\n".encode("ascii")) + os.fsync(descriptor) + yield + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def _run(argv: Sequence[str], *, cwd: Path) -> None: + result = subprocess.run(list(argv), cwd=cwd, check=False) + if result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, list(argv)) + + +def _wait_for_health(url: str, *, timeout_seconds: float) -> None: + deadline = time.monotonic() + max(timeout_seconds, 1.0) + last_error = "health endpoint did not answer" + while time.monotonic() < deadline: + try: + with urlopen(url, timeout=3) as response: # noqa: S310 - URL originates in the validated installation specification. + if 200 <= response.status < 300: + return + last_error = f"health endpoint returned HTTP {response.status}" + except (OSError, URLError) as exc: + last_error = str(exc) + time.sleep(2) + raise ValueError(f"GovOPlaN did not become healthy: {last_error}") + + +def _parse_compose_ps(output: str) -> list[object]: + stripped = output.strip() + if not stripped: + return [] + try: + value = json.loads(stripped) + except json.JSONDecodeError: + rows = [] + for line in stripped.splitlines(): + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + return [{"raw": stripped}] + return rows + return value if isinstance(value, list) else [value] + + +def _now() -> str: + return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace( + "+00:00", "Z" + ) diff --git a/tools/deployment/govoplan_deploy/model.py b/tools/deployment/govoplan_deploy/model.py new file mode 100644 index 0000000..07bdfee --- /dev/null +++ b/tools/deployment/govoplan_deploy/model.py @@ -0,0 +1,482 @@ +"""Installation intent model and validation.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +import ipaddress +import json +from pathlib import Path +import re +from typing import Any, Mapping +from urllib.parse import urlsplit + + +SCHEMA_VERSION = 1 +INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$") +ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$") +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +IMAGE_DIGEST_PATTERN = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") +RFC1918_NETWORKS = tuple( + ipaddress.ip_network(value) + for value in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") +) + +BASE_MODULES = ( + "tenancy", + "organizations", + "identity", + "idm", + "access", + "admin", + "dashboard", + "policy", + "audit", + "docs", + "ops", +) + +FULL_MODULES = ( + *BASE_MODULES, + "addresses", + "dist_lists", + "files", + "mail", + "campaigns", + "calendar", + "poll", + "scheduling", + "connectors", + "datasources", + "dataflow", + "workflow", + "views", + "search", + "risk_compliance", + "postbox", + "evaluation", +) + + +class SpecError(ValueError): + """Raised when an installation specification is malformed.""" + + +@dataclass(frozen=True, slots=True) +class ListenConfig: + address: str + port: int + + +@dataclass(frozen=True, slots=True) +class ReleaseConfig: + channel: str + version: str + manifest_url: str + manifest_sha256: str + api_image: str + web_image: str + + +@dataclass(frozen=True, slots=True) +class ServiceConfig: + mode: str + image: str = "" + url_env: str = "" + + +@dataclass(frozen=True, slots=True) +class StorageConfig: + mode: str + + +@dataclass(frozen=True, slots=True) +class ComponentConfig: + postgres: ServiceConfig + redis: ServiceConfig + mail: ServiceConfig + storage: StorageConfig + + +@dataclass(frozen=True, slots=True) +class InstallationSpec: + schema_version: int + installation_id: str + profile: str + public_url: str + listen: ListenConfig + network_subnet: str + release: ReleaseConfig + components: ComponentConfig + enabled_modules: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + value = asdict(self) + value["enabled_modules"] = list(self.enabled_modules) + return value + + +def default_spec( + *, + installation_id: str = "govoplan-local", + profile: str = "evaluation", + public_url: str = "http://127.0.0.1:8080", + listen_address: str = "127.0.0.1", + listen_port: int = 8080, + postgres_mode: str = "managed", + redis_mode: str = "managed", + mail_mode: str = "disabled", + storage_mode: str = "local", + module_set: str = "base", + api_image: str = "govoplan-api:unpublished", + web_image: str = "govoplan-web:unpublished", + manifest_url: str = "", + manifest_sha256: str = "", + version: str = "unpublished", + channel: str = "stable", +) -> InstallationSpec: + modules = { + "core": (), + "base": BASE_MODULES, + "full": FULL_MODULES, + }.get(module_set) + if modules is None: + raise SpecError("module_set must be core, base, or full") + subnet_octet = 32 + (sum(installation_id.encode("utf-8")) % 192) + raw = { + "schema_version": SCHEMA_VERSION, + "installation_id": installation_id, + "profile": profile, + "public_url": public_url, + "listen": {"address": listen_address, "port": listen_port}, + "network_subnet": f"172.30.{subnet_octet}.0/24", + "release": { + "channel": channel, + "version": version, + "manifest_url": manifest_url, + "manifest_sha256": manifest_sha256, + "api_image": api_image, + "web_image": web_image, + }, + "components": { + "postgres": { + "mode": postgres_mode, + "image": "postgres:16-alpine", + "url_env": "DATABASE_URL", + }, + "redis": { + "mode": redis_mode, + "image": "redis:7-alpine", + "url_env": "REDIS_URL", + }, + "mail": { + "mode": mail_mode, + "image": "greenmail/standalone:2.1.9", + "url_env": "", + }, + "storage": {"mode": storage_mode}, + }, + "enabled_modules": list(modules), + } + return parse_spec(raw) + + +def load_spec(path: Path) -> InstallationSpec: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise SpecError(f"installation specification does not exist: {path}") from exc + except json.JSONDecodeError as exc: + raise SpecError(f"installation specification is not valid JSON: {exc}") from exc + return parse_spec(raw) + + +def parse_spec(raw: object) -> InstallationSpec: + root = _mapping(raw, "installation") + _only_keys( + root, + { + "schema_version", + "installation_id", + "profile", + "public_url", + "listen", + "network_subnet", + "release", + "components", + "enabled_modules", + }, + "installation", + ) + schema_version = _integer(root, "schema_version") + if schema_version != SCHEMA_VERSION: + raise SpecError( + f"schema_version must be {SCHEMA_VERSION}; found {schema_version}" + ) + + installation_id = _string(root, "installation_id") + if not INSTALLATION_ID_PATTERN.fullmatch(installation_id): + raise SpecError( + "installation_id must start with a lowercase letter and contain only " + "lowercase letters, digits, or hyphens (2-48 characters)" + ) + + profile = _choice(root, "profile", {"evaluation", "self-hosted"}) + public_url = _http_url(_string(root, "public_url"), "public_url") + public_parts = urlsplit(public_url) + if profile == "self-hosted" and public_parts.scheme != "https": + raise SpecError("self-hosted public_url must use HTTPS") + + listen_raw = _mapping(root.get("listen"), "listen") + _only_keys(listen_raw, {"address", "port"}, "listen") + listen = ListenConfig( + address=_ip_address(_string(listen_raw, "address"), "listen.address"), + port=_port(_integer(listen_raw, "port"), "listen.port"), + ) + + network_subnet = _network( + _string(root, "network_subnet"), "network_subnet" + ) + release = _release(root.get("release")) + components = _components(root.get("components"), profile=profile) + + enabled_raw = root.get("enabled_modules") + if not isinstance(enabled_raw, list): + raise SpecError("enabled_modules must be an array") + enabled_modules: list[str] = [] + seen_modules: set[str] = set() + for index, value in enumerate(enabled_raw): + if not isinstance(value, str) or not re.fullmatch( + r"[a-z][a-z0-9_]{1,63}", value + ): + raise SpecError( + f"enabled_modules[{index}] must be a canonical module id" + ) + if value in seen_modules: + raise SpecError(f"enabled_modules contains duplicate module {value!r}") + enabled_modules.append(value) + seen_modules.add(value) + + return InstallationSpec( + schema_version=schema_version, + installation_id=installation_id, + profile=profile, + public_url=public_url, + listen=listen, + network_subnet=network_subnet, + release=release, + components=components, + enabled_modules=tuple(enabled_modules), + ) + + +def _release(raw: object) -> ReleaseConfig: + value = _mapping(raw, "release") + _only_keys( + value, + { + "channel", + "version", + "manifest_url", + "manifest_sha256", + "api_image", + "web_image", + }, + "release", + ) + channel = _string(value, "channel") + if not re.fullmatch(r"[a-z][a-z0-9-]{1,31}", channel): + raise SpecError("release.channel must be a canonical channel name") + version = _string(value, "version") + if not version or len(version) > 80 or any(char.isspace() for char in version): + raise SpecError("release.version must be a non-empty version token") + manifest_url = _optional_string(value, "manifest_url") + if manifest_url: + manifest_url = _http_url(manifest_url, "release.manifest_url") + if urlsplit(manifest_url).scheme != "https": + raise SpecError("release.manifest_url must use HTTPS") + manifest_sha256 = _optional_string(value, "manifest_sha256").lower() + if manifest_sha256 and not SHA256_PATTERN.fullmatch(manifest_sha256): + raise SpecError("release.manifest_sha256 must be a lowercase SHA-256 hex digest") + api_image = _image(_string(value, "api_image"), "release.api_image") + web_image = _image(_string(value, "web_image"), "release.web_image") + return ReleaseConfig( + channel=channel, + version=version, + manifest_url=manifest_url, + manifest_sha256=manifest_sha256, + api_image=api_image, + web_image=web_image, + ) + + +def _components(raw: object, *, profile: str) -> ComponentConfig: + value = _mapping(raw, "components") + _only_keys(value, {"postgres", "redis", "mail", "storage"}, "components") + postgres = _service( + value.get("postgres"), + "components.postgres", + modes={"managed", "external"}, + image_required_for={"managed"}, + url_env_required_for={"external"}, + ) + redis = _service( + value.get("redis"), + "components.redis", + modes={"managed", "external", "disabled"}, + image_required_for={"managed"}, + url_env_required_for={"external"}, + ) + mail = _service( + value.get("mail"), + "components.mail", + modes={"disabled", "external-relay", "test-mail"}, + image_required_for={"test-mail"}, + url_env_required_for=set(), + ) + storage_raw = _mapping(value.get("storage"), "components.storage") + _only_keys(storage_raw, {"mode"}, "components.storage") + storage = StorageConfig(mode=_choice(storage_raw, "mode", {"local", "s3"})) + if postgres.url_env != "DATABASE_URL": + raise SpecError("components.postgres.url_env must be DATABASE_URL") + if redis.url_env != "REDIS_URL": + raise SpecError("components.redis.url_env must be REDIS_URL") + if mail.url_env: + raise SpecError("components.mail.url_env must be empty") + if profile == "self-hosted" and redis.mode == "disabled": + raise SpecError("self-hosted installations require managed or external Redis") + if profile == "self-hosted" and mail.mode == "test-mail": + raise SpecError("the test-mail component is restricted to evaluation installs") + return ComponentConfig( + postgres=postgres, + redis=redis, + mail=mail, + storage=storage, + ) + + +def _service( + raw: object, + label: str, + *, + modes: set[str], + image_required_for: set[str], + url_env_required_for: set[str], +) -> ServiceConfig: + value = _mapping(raw, label) + _only_keys(value, {"mode", "image", "url_env"}, label) + mode = _choice(value, "mode", modes) + image = _optional_string(value, "image") + url_env = _optional_string(value, "url_env") + if mode in image_required_for: + image = _image(image, f"{label}.image") + if mode in url_env_required_for: + if not ENV_NAME_PATTERN.fullmatch(url_env): + raise SpecError(f"{label}.url_env must name an environment variable") + return ServiceConfig(mode=mode, image=image, url_env=url_env) + + +def image_is_digest_pinned(value: str) -> bool: + return bool(IMAGE_DIGEST_PATTERN.fullmatch(value)) + + +def image_is_unpublished(value: str) -> bool: + return value.endswith(":unpublished") + + +def _mapping(value: object, label: str) -> Mapping[str, Any]: + if not isinstance(value, dict): + raise SpecError(f"{label} must be an object") + if not all(isinstance(key, str) for key in value): + raise SpecError(f"{label} keys must be strings") + return value + + +def _only_keys(value: Mapping[str, Any], allowed: set[str], label: str) -> None: + unknown = sorted(set(value) - allowed) + if unknown: + raise SpecError(f"{label} contains unknown fields: {', '.join(unknown)}") + + +def _string(value: Mapping[str, Any], key: str) -> str: + result = value.get(key) + if not isinstance(result, str): + raise SpecError(f"{key} must be a string") + return result.strip() + + +def _optional_string(value: Mapping[str, Any], key: str) -> str: + result = value.get(key, "") + if not isinstance(result, str): + raise SpecError(f"{key} must be a string") + return result.strip() + + +def _integer(value: Mapping[str, Any], key: str) -> int: + result = value.get(key) + if isinstance(result, bool) or not isinstance(result, int): + raise SpecError(f"{key} must be an integer") + return result + + +def _choice(value: Mapping[str, Any], key: str, choices: set[str]) -> str: + result = _string(value, key) + if result not in choices: + raise SpecError(f"{key} must be one of: {', '.join(sorted(choices))}") + return result + + +def _http_url(value: str, label: str) -> str: + try: + parsed = urlsplit(value) + parsed.port + except ValueError as exc: + raise SpecError(f"{label} contains an invalid host or port") from exc + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise SpecError(f"{label} must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.fragment or parsed.query: + raise SpecError( + f"{label} must not contain credentials, a query, or a fragment" + ) + if label == "public_url" and parsed.path not in {"", "/"}: + raise SpecError("public_url must address the site root without a path") + return value.rstrip("/") + + +def _image(value: str, label: str) -> str: + if ( + not value + or len(value) > 300 + or any(character.isspace() for character in value) + or value.startswith("-") + ): + raise SpecError(f"{label} must be a valid non-empty OCI image reference") + return value + + +def _ip_address(value: str, label: str) -> str: + try: + return str(ipaddress.ip_address(value)) + except ValueError as exc: + raise SpecError(f"{label} must be an IPv4 or IPv6 address") from exc + + +def _network(value: str, label: str) -> str: + try: + network = ipaddress.ip_network(value, strict=True) + except ValueError as exc: + raise SpecError(f"{label} must be a canonical private IP network") from exc + if ( + network.version != 4 + or not any(network.subnet_of(parent) for parent in RFC1918_NETWORKS) + or not 24 <= network.prefixlen <= 28 + ): + raise SpecError( + f"{label} must be an RFC1918 IPv4 /24 to /28 network" + ) + return str(network) + + +def _port(value: int, label: str) -> int: + if value < 1 or value > 65535: + raise SpecError(f"{label} must be between 1 and 65535") + return value diff --git a/tools/deployment/govoplan_deploy/planning.py b/tools/deployment/govoplan_deploy/planning.py new file mode 100644 index 0000000..04cd2cc --- /dev/null +++ b/tools/deployment/govoplan_deploy/planning.py @@ -0,0 +1,579 @@ +"""Deployment plan and host preflight checks.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +import json +import os +from pathlib import Path +import platform +import shutil +import socket +import stat +import subprocess +from typing import Callable, Mapping, Sequence +from urllib.parse import urlsplit + +from .bundle import ( + BundlePaths, + digest_json, + environment_fingerprint, + read_env, + render_compose, + service_names, +) +from .model import ( + InstallationSpec, + image_is_digest_pinned, + image_is_unpublished, +) + + +@dataclass(frozen=True, slots=True) +class Check: + id: str + level: str + message: str + action: str = "" + + def to_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class PlanAction: + action: str + target: str + detail: str + + def to_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class DeploymentPlan: + installation_id: str + desired_spec_sha256: str + desired_compose_sha256: str + desired_environment_fingerprint: str + actions: tuple[PlanAction, ...] + checks: tuple[Check, ...] + + @property + def blocked(self) -> bool: + return any(check.level == "error" for check in self.checks) + + def to_dict(self) -> dict[str, object]: + return { + "installation_id": self.installation_id, + "desired_spec_sha256": self.desired_spec_sha256, + "desired_compose_sha256": self.desired_compose_sha256, + "desired_environment_fingerprint": ( + self.desired_environment_fingerprint + ), + "blocked": self.blocked, + "actions": [action.to_dict() for action in self.actions], + "checks": [check.to_dict() for check in self.checks], + } + + +CommandRunner = Callable[[Sequence[str], Path], subprocess.CompletedProcess[str]] + + +def build_plan( + spec: InstallationSpec, + paths: BundlePaths, + *, + include_host_checks: bool = True, + command_runner: CommandRunner | None = None, +) -> DeploymentPlan: + compose = render_compose(spec) + desired_services = set(service_names(spec)) + previous = _read_receipt(paths.receipt) + previous_services = { + str(item) for item in previous.get("services", []) if isinstance(item, str) + } + previous_spec_digest = previous.get("spec_sha256") + previous_compose_digest = previous.get("compose_sha256") + previous_environment_fingerprint = previous.get("environment_fingerprint") + spec_digest = digest_json(spec.to_dict()) + compose_digest = digest_json(compose) + environment_digest = environment_fingerprint(read_env(paths.env)) + + actions: list[PlanAction] = [] + if not previous: + actions.append( + PlanAction( + "create", + "installation", + "Create the first deployment revision.", + ) + ) + if previous_spec_digest != spec_digest: + actions.append( + PlanAction( + "reconfigure", + "installation", + "Reconcile runtime configuration with installation.json.", + ) + ) + if previous_compose_digest != compose_digest: + actions.append( + PlanAction( + "render", + "compose", + "Render a new deterministic Compose definition.", + ) + ) + if previous_environment_fingerprint != environment_digest: + actions.append( + PlanAction( + "reconfigure", + "environment", + "Reconcile changed secret bindings or runtime environment values.", + ) + ) + for name in sorted(desired_services - previous_services): + actions.append(PlanAction("start", name, "Start the selected service.")) + for name in sorted(previous_services - desired_services): + actions.append( + PlanAction( + "remove", + name, + "Remove the service container; retained volumes are not deleted.", + ) + ) + if ( + previous + and previous_spec_digest == spec_digest + and previous_compose_digest == compose_digest + and previous_environment_fingerprint == environment_digest + and previous_services == desired_services + ): + actions.append( + PlanAction("noop", "installation", "Desired state matches the last receipt.") + ) + + checks = list(static_checks(spec, paths)) + if include_host_checks: + checks.extend(host_checks(spec, paths, command_runner=command_runner)) + return DeploymentPlan( + installation_id=spec.installation_id, + desired_spec_sha256=spec_digest, + desired_compose_sha256=compose_digest, + desired_environment_fingerprint=environment_digest, + actions=tuple(actions), + checks=tuple(checks), + ) + + +def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, ...]: + checks: list[Check] = [] + images = { + "release.api_image": spec.release.api_image, + "release.web_image": spec.release.web_image, + } + if spec.components.postgres.mode == "managed": + images["components.postgres.image"] = spec.components.postgres.image + if spec.components.redis.mode == "managed": + images["components.redis.image"] = spec.components.redis.image + if spec.components.mail.mode == "test-mail": + images["components.mail.image"] = spec.components.mail.image + + for label, image in images.items(): + if image_is_unpublished(image): + checks.append( + Check( + f"{label}.published", + "error", + f"{label} still uses the unpublished placeholder.", + "Set an available image reference before apply.", + ) + ) + elif not image_is_digest_pinned(image): + level = "error" if spec.profile == "self-hosted" else "warning" + checks.append( + Check( + f"{label}.digest", + level, + f"{label} is not pinned by OCI digest.", + "Use an image@sha256:... reference from a verified distribution.", + ) + ) + else: + checks.append( + Check( + f"{label}.digest", + "ok", + f"{label} is pinned by OCI digest.", + ) + ) + + if spec.release.manifest_url and spec.release.manifest_sha256: + checks.append( + Check( + "release.manifest", + "ok", + "A distribution manifest URL and expected digest are recorded.", + ) + ) + else: + checks.append( + Check( + "release.manifest", + "error" if spec.profile == "self-hosted" else "warning", + "No verified distribution manifest is recorded.", + "Use a published signed distribution manifest for self-hosted apply.", + ) + ) + if spec.profile == "self-hosted": + checks.append( + Check( + "release.signature_verification", + "error", + "Signed distribution-manifest verification is not implemented in the deployer yet.", + "Use the published verifier/bootstrap slice before a production apply.", + ) + ) + checks.append( + Check( + "modules.image_composition", + "error" if spec.enabled_modules else "warning", + "The selected module set is not yet verified against image package contents.", + "Use the signed distribution composition evidence before production apply.", + ) + ) + + values = read_env(paths.env) + required = {"MASTER_KEY_B64", "DATABASE_URL"} + if spec.components.redis.mode != "disabled": + required.add("REDIS_URL") + if spec.components.storage.mode == "s3": + required.update( + { + "FILE_STORAGE_S3_ENDPOINT_URL", + "FILE_STORAGE_S3_REGION", + "FILE_STORAGE_S3_ACCESS_KEY_ID", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY", + "FILE_STORAGE_S3_BUCKET", + } + ) + missing = sorted(name for name in required if not values.get(name)) + checks.append( + Check( + "secrets.required", + "error" if missing else "ok", + ( + "Missing required secret values: " + ", ".join(missing) + if missing + else "Required runtime secret references are populated." + ), + "Re-run configure with the required external service values." if missing else "", + ) + ) + if paths.env.exists(): + mode = stat.S_IMODE(paths.env.stat().st_mode) + checks.append( + Check( + "secrets.permissions", + "error" if mode & 0o077 else "ok", + ( + f"{paths.env.name} has unsafe mode {mode:04o}." + if mode & 0o077 + else f"{paths.env.name} is private ({mode:04o})." + ), + f"Run chmod 600 {paths.env}" if mode & 0o077 else "", + ) + ) + if spec.profile == "self-hosted": + checks.append( + Check( + "bootstrap.administrator", + "warning", + "Production first-administrator enrollment is not automated yet.", + "Use the controlled one-time administrator procedure until the enrollment slice lands.", + ) + ) + if spec.components.mail.mode == "external-relay": + checks.append( + Check( + "mail.provisioning", + "warning", + "The external relay is selected but Mail profile seeding remains an explicit post-install configuration task.", + "Create the reusable server and credential profile in Mail after first login.", + ) + ) + if spec.components.storage.mode == "local" and spec.profile == "self-hosted": + checks.append( + Check( + "storage.local", + "warning", + "Local file storage is suitable for one host but prevents stateless API scale-out.", + "Include files-data in backup/restore drills or configure S3 storage.", + ) + ) + return tuple(checks) + + +def host_checks( + spec: InstallationSpec, + paths: BundlePaths, + *, + command_runner: CommandRunner | None = None, +) -> tuple[Check, ...]: + checks: list[Check] = [] + machine = platform.machine().lower() + supported = machine in {"x86_64", "amd64", "aarch64", "arm64"} + checks.append( + Check( + "host.architecture", + "ok" if supported else "error", + f"Host architecture is {machine or 'unknown'}.", + "Use a supported amd64 or arm64 host." if not supported else "", + ) + ) + + memory_bytes = _memory_bytes() + if memory_bytes is not None: + minimum = 4 * 1024**3 + checks.append( + Check( + "host.memory", + "ok" if memory_bytes >= minimum else "warning", + f"Host memory is approximately {memory_bytes / 1024**3:.1f} GiB.", + "Use at least 4 GiB for the base container profile." + if memory_bytes < minimum + else "", + ) + ) + cpu_count = os.cpu_count() + if cpu_count is not None: + checks.append( + Check( + "host.cpu", + "ok" if cpu_count >= 2 else "warning", + f"Host reports {cpu_count} logical CPU(s).", + "Use at least two logical CPUs for the base container profile." + if cpu_count < 2 + else "", + ) + ) + entropy = _entropy_available() + if entropy is not None: + checks.append( + Check( + "host.entropy", + "ok" if entropy >= 256 else "warning", + f"Kernel entropy availability is {entropy}.", + "Wait for or provide sufficient host entropy before generating production keys." + if entropy < 256 + else "", + ) + ) + free_bytes = shutil.disk_usage(paths.root).free + minimum_free = 10 * 1024**3 + checks.append( + Check( + "host.disk", + "ok" if free_bytes >= minimum_free else "warning", + f"Free installation filesystem space is approximately {free_bytes / 1024**3:.1f} GiB.", + "Keep at least 10 GiB free before pulling images and creating data volumes." + if free_bytes < minimum_free + else "", + ) + ) + + docker = shutil.which("docker") + if docker is None: + checks.append( + Check( + "host.compose", + "error", + "Docker CLI is not installed or not on PATH.", + "Install Docker Engine with the Compose v2 plugin.", + ) + ) + else: + runner = command_runner or _run_command + result = runner((docker, "compose", "version", "--short"), paths.root) + checks.append( + Check( + "host.compose", + "ok" if result.returncode == 0 else "error", + ( + f"Docker Compose is available ({result.stdout.strip()})." + if result.returncode == 0 + else "Docker Compose v2 is not available." + ), + "Install or enable the Docker Compose v2 plugin." + if result.returncode != 0 + else "", + ) + ) + daemon = runner( + (docker, "info", "--format", "{{.ServerVersion}}"), + paths.root, + ) + checks.append( + Check( + "host.container_runtime", + "ok" if daemon.returncode == 0 else "error", + ( + f"Docker daemon is reachable ({daemon.stdout.strip()})." + if daemon.returncode == 0 + else "Docker CLI cannot reach the Docker daemon." + ), + "Start Docker and grant this operator access to the daemon." + if daemon.returncode != 0 + else "", + ) + ) + + values = read_env(paths.env) + if spec.components.postgres.mode == "external": + checks.append( + _endpoint_check( + "external.postgres", + "External PostgreSQL", + values.get("DATABASE_URL", ""), + default_ports={"postgresql": 5432, "postgresql+psycopg": 5432}, + ) + ) + if spec.components.redis.mode == "external": + checks.append( + _endpoint_check( + "external.redis", + "External Redis", + values.get("REDIS_URL", ""), + default_ports={"redis": 6379, "rediss": 6379}, + ) + ) + if spec.components.storage.mode == "s3": + checks.append( + _endpoint_check( + "external.s3", + "S3-compatible storage", + values.get("FILE_STORAGE_S3_ENDPOINT_URL", ""), + default_ports={"http": 80, "https": 443}, + ) + ) + + receipt = _read_receipt(paths.receipt) + previous_listen = receipt.get("listen") + desired_listen = { + "address": spec.listen.address, + "port": spec.listen.port, + } + if not receipt or previous_listen != desired_listen: + available = _port_available(spec.listen.address, spec.listen.port) + checks.append( + Check( + "host.listen_port", + "ok" if available else "error", + ( + f"Listen endpoint {spec.listen.address}:{spec.listen.port} is available." + if available + else f"Listen endpoint {spec.listen.address}:{spec.listen.port} is already in use." + ), + "Choose another listen port or stop the conflicting service." + if not available + else "", + ) + ) + return tuple(checks) + + +def _read_receipt(path: Path) -> Mapping[str, object]: + if not path.exists(): + return {} + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def _memory_bytes() -> int | None: + try: + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("MemTotal:"): + return int(line.split()[1]) * 1024 + except (OSError, ValueError, IndexError): + return None + return None + + +def _entropy_available() -> int | None: + try: + return int( + Path("/proc/sys/kernel/random/entropy_avail") + .read_text(encoding="utf-8") + .strip() + ) + except (OSError, ValueError): + return None + + +def _port_available(address: str, port: int) -> bool: + family = socket.AF_INET6 if ":" in address else socket.AF_INET + with socket.socket(family, socket.SOCK_STREAM) as handle: + try: + handle.bind((address, port)) + except OSError: + return False + return True + + +def _endpoint_check( + check_id: str, + label: str, + url: str, + *, + default_ports: Mapping[str, int], +) -> Check: + try: + parsed = urlsplit(url) + host = parsed.hostname + port = parsed.port or default_ports.get(parsed.scheme) + except ValueError as exc: + return Check( + check_id, + "error", + f"{label} endpoint is invalid: {exc}", + "Correct the private endpoint binding and rerun doctor.", + ) + if not host or port is None: + return Check( + check_id, + "error", + f"{label} endpoint has no usable host and port.", + "Correct the private endpoint binding and rerun doctor.", + ) + try: + with socket.create_connection((host, port), timeout=1.5): + pass + except OSError as exc: + return Check( + check_id, + "error", + f"{label} is not reachable at {host}:{port}: {exc}", + "Check DNS, routing, firewall, service health, and the selected endpoint.", + ) + return Check( + check_id, + "ok", + f"{label} is reachable at {host}:{port}.", + ) + + +def _run_command( + argv: Sequence[str], cwd: Path +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(argv), + cwd=cwd, + check=False, + capture_output=True, + text=True, + timeout=10, + )