From 908090dd0f8da1f6c8d1c19a49a68b2d21f66331 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 31 Jul 2026 02:48:55 +0200 Subject: [PATCH] Add scalable deployment planning and guided releases --- .gitea/workflows/security-audit.yml | 14 - README.md | 4 + ...NSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md | 66 +- docs/RELEASE_CONSOLE.md | 23 +- docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md | 156 +++++ docs/installation-spec.schema.json | 51 ++ tests/test_deployment_installer.py | 198 +++++- tests/test_release_plan_guidance.py | 22 + tools/deployment/govoplan_deploy/bundle.py | 210 ++++++- tools/deployment/govoplan_deploy/cli.py | 222 ++++++- tools/deployment/govoplan_deploy/model.py | 137 ++++- tools/deployment/govoplan_deploy/planning.py | 65 +- tools/release/webui/index.html | 571 +++++++++++++++--- 13 files changed, 1511 insertions(+), 228 deletions(-) create mode 100644 docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md diff --git a/.gitea/workflows/security-audit.yml b/.gitea/workflows/security-audit.yml index 912024f..2e2cdfc 100644 --- a/.gitea/workflows/security-audit.yml +++ b/.gitea/workflows/security-audit.yml @@ -24,20 +24,6 @@ jobs: run: | git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan" git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "ssh://git@git.add-ideas.de/GovOPlaN/govoplan" - - name: Configure SSH for repository bootstrap - env: - GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then - echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh repository bootstrap." - exit 1 - fi - printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts - chmod 600 ~/.ssh/known_hosts - name: Bootstrap GovOPlaN repositories working-directory: govoplan run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https diff --git a/README.md b/README.md index 5f49a9e..70b9c9f 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,8 @@ Create and validate a private, declarative installation bundle: The current executable slice and remaining production gates are documented in [Installation and Deployment Architecture](docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md). +Same-host replica balancing and the multi-host promotion boundary are documented +in [Scaling and Multi-Host Deployment](docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md). ## Configuration @@ -189,6 +191,8 @@ installation, scale-out, and reversible environment promotion is defined in The corresponding host deployment compiler, managed/external component choices, reconfiguration semantics, and safe Web update boundary are defined in [Installation and Deployment Architecture](docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md). +The concrete replica, worker-node, load-balancer, and shared-state topology is +defined in [Scaling and Multi-Host Deployment](docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md). The first Campaign-centric capability and infrastructure fit assessment is in `docs/CAPABILITY_AND_INFRASTRUCTURE_FIT.md`. Its rerun tooling can collect and verify a bounded installed composition; target, provider and production claims diff --git a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index 3718615..d41fb11 100644 --- a/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -23,7 +23,12 @@ It currently supports: - managed, external, or evaluation-only disabled Redis; - disabled mail, an external relay declaration, or an evaluation-only GreenMail service; -- durable local file storage or external S3-compatible storage; +- durable local file storage, managed single-node Garage S3, or external + S3-compatible storage; +- an explicit HAProxy service that load-balances configured WebUI and API + replicas without access to the Docker socket; +- declarative API, WebUI, and worker replica counts while keeping migrations + and the scheduler singleton; - Core, base, or full initial module selections; - deterministic Compose JSON accepted by Compose v2; - generated secrets stored in a private `0600` file; @@ -47,7 +52,11 @@ Create a local evaluation bundle: --profile evaluation \ --postgres managed \ --redis managed \ + --storage garage \ --mail test-mail \ + --api-replicas 2 \ + --web-replicas 2 \ + --worker-replicas 2 \ --module-set base ``` @@ -74,6 +83,8 @@ The private installation directory contains: | `installation.json` | Versioned, non-secret desired state | | `secrets.env` | Deployment-local secrets and external service bindings | | `compose.json` | Deterministic generated Compose definition | +| `garage.toml` | Non-secret managed Garage server configuration | +| `load-balancer.cfg` | Non-secret HAProxy WebUI/API discovery configuration | | `plan.json` | Latest desired-state diff and readiness findings | | `receipt.json` | Last successfully applied immutable identities | | `.deployment.lock` | Same-host operation exclusion | @@ -126,7 +137,9 @@ a production distribution: 5. **Deployment agent.** Web updates need a separate privileged reconciler with a typed command allowlist. The API and browser must never receive the Docker socket or arbitrary shell access. -6. **Ingress and certificates.** A self-hosted profile needs an explicit choice +6. **Ingress and certificates.** The managed HAProxy service provides HTTP + load balancing inside the deployment boundary; it does not issue or renew + certificates. A self-hosted profile still needs an explicit choice between an existing reverse proxy and a supported managed ingress, including trusted-proxy boundaries, TLS certificate issuance, renewal, and health probing through the public route. @@ -185,11 +198,49 @@ presenting a container as a complete mail service. ### File Storage `local` uses a durable Compose volume and is appropriate for one-host -installations. `s3` requires endpoint, region, access key, secret key, and -bucket values. Self-hosted S3 endpoints must use HTTPS. +installations. `garage` provisions Garage 2.3 in its supported single-node +bootstrap mode, generates a private application key and bucket, and connects +the Files S3 backend to the exact installer-owned internal endpoint. The +deployment-only trust marker cannot authorize another S3 host; arbitrary +external SDK endpoints remain fail-closed until peer pinning is implemented. +Garage metadata and object data use separate persistent volumes. `s3` requires +an external endpoint, region, access key, secret key, and bucket values. +Self-hosted external S3 endpoints must use HTTPS. Local storage must be included in backup and restore drills. Horizontal API or -worker scale-out requires shared/object storage. +worker scale-out requires shared/object storage. The managed Garage profile is +persistent but has no data redundancy; availability-sensitive installations +must use a tested multi-node Garage cluster or another external S3 service. + +### Load Balancing And Replicas + +The generated Compose topology publishes only `load-balancer`. HAProxy uses +Docker DNS service discovery to distribute public traffic across WebUI replicas +and WebUI API proxy traffic across API replicas. The WebUI and API services do +not publish host ports. HAProxy has no Docker socket and discovers only the +bounded replica slots rendered into `load-balancer.cfg`. + +Replica counts are desired state: + +```sh +./.venv/bin/python tools/deployment/govoplan-deploy.py configure \ + --directory /tmp/govoplan-evaluation \ + --api-replicas 3 \ + --web-replicas 2 \ + --worker-replicas 4 +./.venv/bin/python tools/deployment/govoplan-deploy.py apply \ + --directory /tmp/govoplan-evaluation +``` + +Workers are queue consumers, so they are scaled through Redis rather than put +behind an HTTP load balancer. The migration runner and Celery scheduler remain +singletons. Multiple API replicas are rejected when Redis is disabled because +distributed throttling and queued work cannot then be shared correctly. + +This is same-host scaling. Docker Compose uses a bridge network and does not +place containers on another machine. See +[Scaling And Multi-Host Deployment](SCALING_AND_MULTI_HOST_DEPLOYMENT.md) for +the supported topology and promotion path. ## Reconfiguration Semantics @@ -207,6 +258,8 @@ applied state. `plan` compares their canonical hashes and service sets. - Managed-to-external transitions require the new endpoint in the same operation. - Migrations run as a one-shot service before API/worker replacement. +- The first upgrade from a direct WebUI host port stops that legacy WebUI + container immediately before HAProxy claims the same endpoint. - Health must recover before a new receipt is committed. This is sufficient for one-host reconciliation. Production updates additionally @@ -261,7 +314,8 @@ Run the focused tests: ``` The tests cover profile restrictions, secret persistence, external endpoint -requirements, S3 policy, Compose service selection, secret non-disclosure, +requirements, managed Garage bootstrap, S3 policy, replica validation, HAProxy +discovery configuration, Compose service selection, secret non-disclosure, service-specific environment isolation, private file modes, external endpoint preflight, first-plan generation, apply ordering, and receipt-based idempotency. diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index 44cc74b..5338d78 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -67,6 +67,24 @@ shows the dry-run commands for the selected rows, and `Generate Candidate` creates a signed catalog candidate that advances only selected repositories that already have a catalog entry. +The full-width **Release Workflow** guide projects the server state into seven +operator phases: Inspect, Targets, Validate, Source, Package, Publish, and +Verify. It does not maintain a second workflow state. Completed, current, +blocked, locked, and unavailable phases are derived from the dashboard, +selective plan, and durable run record. The next-action panel opens the exact +section or durable step that needs attention. Changing a channel, target +version, repository selection, or release gate detaches the browser from the +current run and invalidates the draft plan; the persisted run remains available +from the saved-run selector. Problems in unselected repositories remain visible +as workspace notices but do not lock an unrelated release; the selective plan +is the authority for blockers in the selected repository set. + +Installation verification remains an explicit unavailable phase after a +durable run completes. It is currently enforced by release-integration CI, +rather than being presented as a successful local-console step. This prevents +the guide from treating source publication as proof that the published package +can be installed and started. + `Build Plan` also returns structured release-gate findings for each selected repository. The plan names the recommended next action and gives an explicit remediation for source-version, lockfile, Core WebUI composition, Git state, and @@ -226,8 +244,9 @@ other `/api/` route: - `POST /api/release-runs/{run_id}/steps/{step_id}/preview` provides the non-mutating preview for receipt-bound catalog publication. -Run-storage errors are confined to the Durable Run State card; dashboard and -release-preview collection continue and show the bounded storage remediation. +Run-storage errors are confined to the Durable Release Run section; dashboard +and release-preview collection continue and the workflow guide points to the +bounded storage remediation. The run record is execution evidence only for a supported step whose durable claim and bounded result receipt were persisted. The console never infers diff --git a/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md new file mode 100644 index 0000000..b31bf62 --- /dev/null +++ b/docs/SCALING_AND_MULTI_HOST_DEPLOYMENT.md @@ -0,0 +1,156 @@ +# Scaling And Multi-Host Deployment + +## What Is Implemented + +The default installer now supports explicit same-host horizontal scaling: + +- one HAProxy container accepts the published HTTP endpoint; +- one or more WebUI containers serve assets and proxy API requests; +- one or more API containers serve normal domain traffic; +- one or more Celery workers consume shared Redis queues; +- one migration job runs before replacement; +- exactly one Celery scheduler runs; +- PostgreSQL, Redis, and file storage are shared by every runtime replica. + +HAProxy discovers Compose replicas through Docker's internal DNS and performs +health-checked round-robin WebUI balancing and least-connection API balancing. +It does not mount the Docker socket. Replica counts live in +`installation.json`, so `configure`, `plan`, `apply`, and the receipt agree on +the desired topology. + +This is not multi-host scheduling. Docker Compose's bridge network belongs to +one Docker Engine. Adding a second machine requires an orchestrator or +deployment manager that can place equivalent role definitions on multiple +hosts. + +## Recommended Topologies + +### One Host + +Use the generated Compose bundle: + +```text +client + -> external TLS proxy, when required + -> managed HAProxy + -> WebUI replica(s) + -> managed HAProxy API listener + -> API replica(s) + +API/worker/scheduler + -> PostgreSQL + -> Redis + -> local volume, managed Garage, or external S3 +``` + +This improves concurrency and permits rolling process replacement, but the host +and every managed stateful component remain single failure domains. Two API +containers on one failed host do not provide host-level availability. + +### Multiple Hosts + +Use Kubernetes, Nomad, Docker Swarm, or another reviewed orchestrator. Do not +extend the Compose installer into a proprietary scheduler. The target topology +is: + +```text +public TLS ingress/load balancer + -> WebUI replicas on at least two nodes + -> internal API service/load balancer + -> API replicas on at least two nodes + +queue-specific worker pools on worker nodes +one fenced scheduler +one fenced migration/deployment job + +shared PostgreSQL +shared Redis +multi-node Garage or external S3 +shared secret/config provider +central logs, metrics, and health alerts +``` + +All API and worker nodes must receive the same immutable software composition, +`MASTER_KEY_B64`, database URL, Redis URL, enabled-module graph, and object +storage binding. Node-local file storage is not valid in this topology. + +## Adding Capacity + +### API And WebUI + +Increase API replicas for measured request concurrency, CPU saturation, or +latency after checking database load. Increase WebUI replicas for static asset +and proxy capacity. No sticky session should be required because durable +sessions and throttling use shared services, but this must remain covered by +multi-replica integration tests. + +Every added API process also adds database connections. Set the application +pool size and the total replica ceiling against PostgreSQL's connection budget; +adding replicas can otherwise reduce throughput. + +### Workers + +Workers are not placed behind a load balancer. They compete for jobs on shared +Redis queues. Add workers by queue and cap each pool according to the external +system it calls. SMTP, IMAP, directory, and connector jobs often reach provider +rate limits before CPU limits. + +Use distinct pools when load warrants it: + +- mail send and Sent-folder append; +- notifications; +- calendar and connector synchronization; +- dataflow/workflow execution; +- reporting and export; +- platform events and default work. + +Before stopping a worker, mark it draining, stop new claims, and let or safely +requeue active jobs. Worker registration and drain control remain part of the +larger platform scale-out story. + +### Stateful Services + +- PostgreSQL needs backups, restore drills, connection limits, and an HA/failover + design appropriate to the service level. +- Redis needs persistence and, for high availability, a supported failover + topology. Queue loss is not equivalent to a harmless cache loss. +- Managed Garage from the Compose installer is single-node. For redundant + storage, deploy a multi-node Garage cluster separately and select external + `s3`, or use another compatible object store. +- The scheduler stays at one replica until distributed leader election or a + fenced lease is implemented. +- Migrations and module lifecycle mutations remain one-at-a-time operations. + +## Promotion Path + +1. Measure concurrent users, request latency, database query time, connection + use, queue age, worker saturation, storage latency, and external throttling. +2. Move files to managed Garage or external S3 before introducing independent + runtime hosts. +3. Keep PostgreSQL and Redis external to stateless runtime nodes, or deploy + their reviewed HA operators. +4. Publish immutable API/WebUI images and one versioned configuration/secret + contract. +5. Translate the generated role commands, environment allowlists, health + checks, and singleton constraints into the chosen orchestrator. +6. Put API and WebUI replicas behind health-aware services and a TLS ingress. +7. Add queue-specific workers with fixed upper bounds. +8. Prove replica loss, rolling replacement, job redelivery, session continuity, + migration exclusion, backup restore, and storage-node loss before claiming + high availability. + +## Remaining Platform Work + +The one-host installer does not yet provide: + +- a Kubernetes, Nomad, or Swarm deployment export/profile; +- distributed deployment locks and fenced scheduler leadership; +- worker registration, composition-skew reporting, and drain controls; +- managed PostgreSQL, Redis, or multi-node Garage high availability; +- autoscaling policy; +- managed TLS certificate issuance and renewal; +- measured multi-replica and failover integration evidence. + +These are separate production capabilities. The local load balancer and +replica model provide the contract they should implement, but they do not by +themselves make a cluster highly available. diff --git a/docs/installation-spec.schema.json b/docs/installation-spec.schema.json index d00ef81..0314ead 100644 --- a/docs/installation-spec.schema.json +++ b/docs/installation-spec.schema.json @@ -124,10 +124,44 @@ "mode": { "enum": [ "local", + "garage", "s3" ] + }, + "image": { + "type": "string", + "maxLength": 300 } } + }, + "load_balancer": { + "$ref": "#/$defs/load_balancer" + } + } + }, + "replicas": { + "type": "object", + "additionalProperties": false, + "required": [ + "api", + "web", + "worker" + ], + "properties": { + "api": { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + "web": { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + "worker": { + "type": "integer", + "minimum": 0, + "maximum": 128 } } }, @@ -222,6 +256,23 @@ } } ] + }, + "load_balancer": { + "allOf": [ + { + "$ref": "#/$defs/service" + }, + { + "properties": { + "mode": { + "const": "managed" + }, + "url_env": { + "const": "" + } + } + } + ] } }, "allOf": [ diff --git a/tests/test_deployment_installer.py b/tests/test_deployment_installer.py index 171c5b6..d6d13f4 100644 --- a/tests/test_deployment_installer.py +++ b/tests/test_deployment_installer.py @@ -26,9 +26,10 @@ from govoplan_deploy.bundle import ( # noqa: E402 read_env, reconcile_runtime_environment, render_compose, + render_load_balancer_config, write_env, ) -from govoplan_deploy.cli import main # noqa: E402 +from govoplan_deploy.cli import _receipt_uses_direct_web_port, main # noqa: E402 import govoplan_deploy.cli as deployment_cli # noqa: E402 from govoplan_deploy.model import ( # noqa: E402 SpecError, @@ -57,11 +58,15 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertIn("postgres", compose["services"]) self.assertIn("redis", compose["services"]) self.assertIn("worker", compose["services"]) + self.assertIn("load-balancer", compose["services"]) self.assertNotIn("test-mail", compose["services"]) self.assertEqual( ["127.0.0.1:8080:8080"], - compose["services"]["web"]["ports"], + compose["services"]["load-balancer"]["ports"], ) + self.assertNotIn("ports", compose["services"]["web"]) + self.assertEqual(1, compose["services"]["api"]["scale"]) + self.assertEqual(1, compose["services"]["web"]["scale"]) def test_disabled_redis_removes_workers_and_sets_single_process_acknowledgement( self, @@ -74,9 +79,7 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertNotIn("worker", compose["services"]) self.assertNotIn("scheduler", compose["services"]) self.assertEqual("false", values["CELERY_ENABLED"]) - self.assertEqual( - "true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] - ) + self.assertEqual("true", values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"]) def test_self_hosted_rejects_insecure_or_test_only_choices(self) -> None: with self.assertRaisesRegex(SpecError, "must use HTTPS"): @@ -149,9 +152,7 @@ class DeploymentInstallerTests(unittest.TestCase): "REDIS_URL": "rediss://:secret@redis.example.test/0", }, ) - self.assertEqual( - "rediss://:secret@redis.example.test/0", values["REDIS_URL"] - ) + self.assertEqual("rediss://:secret@redis.example.test/0", values["REDIS_URL"]) def test_s3_requires_complete_private_configuration_and_https_in_production( self, @@ -169,9 +170,7 @@ class DeploymentInstallerTests(unittest.TestCase): } self.assertEqual( "s3", - initial_secrets(evaluation, supplied=supplied)[ - "FILE_STORAGE_BACKEND" - ], + initial_secrets(evaluation, supplied=supplied)["FILE_STORAGE_BACKEND"], ) production = default_spec( profile="self-hosted", @@ -181,6 +180,92 @@ class DeploymentInstallerTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "must use HTTPS"): initial_secrets(production, supplied=supplied) + def test_managed_garage_bootstraps_private_s3_storage(self) -> None: + spec = default_spec(storage_mode="garage") + values = initial_secrets(spec) + compose = render_compose(spec) + + self.assertEqual("s3", values["FILE_STORAGE_BACKEND"]) + self.assertEqual("true", values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"]) + self.assertEqual( + "http://garage:3900", + values["FILE_STORAGE_S3_ENDPOINT_URL"], + ) + self.assertEqual("garage", values["FILE_STORAGE_S3_REGION"]) + self.assertEqual( + values["GARAGE_DEFAULT_ACCESS_KEY"], + values["FILE_STORAGE_S3_ACCESS_KEY_ID"], + ) + self.assertEqual( + values["GARAGE_DEFAULT_SECRET_KEY"], + values["FILE_STORAGE_S3_SECRET_ACCESS_KEY"], + ) + self.assertTrue(values["GARAGE_DEFAULT_ACCESS_KEY"].startswith("GK")) + self.assertEqual(34, len(values["GARAGE_DEFAULT_ACCESS_KEY"])) + self.assertEqual(64, len(values["GARAGE_DEFAULT_SECRET_KEY"])) + self.assertIn("garage", compose["services"]) + self.assertIn("garage-meta", compose["volumes"]) + self.assertIn("garage-data", compose["volumes"]) + self.assertEqual( + ["/garage", "server", "--single-node", "--default-bucket"], + compose["services"]["garage"]["command"], + ) + self.assertNotIn( + values["GARAGE_DEFAULT_SECRET_KEY"], + json.dumps(compose), + ) + + reconciled = reconcile_runtime_environment(spec, values) + self.assertEqual( + values["GARAGE_DEFAULT_ACCESS_KEY"], + reconciled["GARAGE_DEFAULT_ACCESS_KEY"], + ) + self.assertEqual( + values["GARAGE_RPC_SECRET"], + reconciled["GARAGE_RPC_SECRET"], + ) + + def test_replica_counts_drive_compose_and_load_balancer_discovery(self) -> None: + spec = default_spec( + storage_mode="garage", + api_replicas=3, + web_replicas=2, + worker_replicas=4, + ) + compose = render_compose(spec) + config = render_load_balancer_config(spec) + + self.assertEqual(3, compose["services"]["api"]["scale"]) + self.assertEqual(2, compose["services"]["web"]["scale"]) + self.assertEqual(4, compose["services"]["worker"]["scale"]) + self.assertEqual( + "http://load-balancer:8000", + compose["services"]["web"]["environment"]["GOVOPLAN_API_UPSTREAM"], + ) + self.assertIn("server-template web- 2 web:8080", config) + self.assertIn("server-template api- 3 api:8000", config) + + def test_multi_replica_runtime_requires_shared_redis(self) -> None: + with self.assertRaisesRegex(SpecError, "multiple API replicas"): + default_spec(redis_mode="disabled", api_replicas=2) + with self.assertRaisesRegex(SpecError, "worker must be 0"): + default_spec(redis_mode="disabled", worker_replicas=1) + with self.assertRaisesRegex(SpecError, "at least 1"): + default_spec(redis_mode="managed", worker_replicas=0) + + def test_legacy_spec_defaults_new_topology_fields(self) -> None: + raw = default_spec().to_dict() + raw.pop("replicas") + raw["components"].pop("load_balancer") + raw["components"]["storage"].pop("image") + + parsed = parse_spec(raw) + + self.assertEqual(1, parsed.replicas.api) + self.assertEqual(1, parsed.replicas.web) + self.assertEqual(1, parsed.replicas.worker) + self.assertEqual("managed", parsed.components.load_balancer.mode) + def test_compose_contains_no_secret_values(self) -> None: spec = default_spec() values = initial_secrets(spec) @@ -199,9 +284,7 @@ class DeploymentInstallerTests(unittest.TestCase): postgres_environment = render_compose(spec)["services"]["postgres"][ "environment" ] - redis_environment = render_compose(spec)["services"]["redis"][ - "environment" - ] + redis_environment = render_compose(spec)["services"]["redis"]["environment"] self.assertEqual( {"POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD"}, set(postgres_environment), @@ -244,9 +327,7 @@ class DeploymentInstallerTests(unittest.TestCase): receipt = { "spec_sha256": first.desired_spec_sha256, "compose_sha256": first.desired_compose_sha256, - "environment_fingerprint": ( - first.desired_environment_fingerprint - ), + "environment_fingerprint": (first.desired_environment_fingerprint), "services": list(render_compose(spec)["services"]), } atomic_write(paths.receipt, canonical_json(receipt), mode=0o600) @@ -264,9 +345,7 @@ class DeploymentInstallerTests(unittest.TestCase): first_spec = default_spec(mail_mode="test-mail") first_environment = initial_secrets(first_spec) write_env(paths.env, first_environment) - first_plan = build_plan( - first_spec, paths, include_host_checks=False - ) + first_plan = build_plan(first_spec, paths, include_host_checks=False) atomic_write( paths.receipt, canonical_json( @@ -287,9 +366,7 @@ class DeploymentInstallerTests(unittest.TestCase): paths.env, reconcile_runtime_environment(second_spec, first_environment), ) - second_plan = build_plan( - second_spec, paths, include_host_checks=False - ) + second_plan = build_plan(second_spec, paths, include_host_checks=False) removed = { action.target for action in second_plan.actions @@ -385,8 +462,11 @@ class DeploymentInstallerTests(unittest.TestCase): "compose.json", "plan.json", ): + self.assertEqual(0o600, stat.S_IMODE((root / name).stat().st_mode)) + for name in ("garage.toml", "load-balancer.cfg"): self.assertEqual( - 0o600, stat.S_IMODE((root / name).stat().st_mode) + 0o644, + stat.S_IMODE((root / name).stat().st_mode), ) values = read_env(root / "secrets.env") self.assertTrue(values["MASTER_KEY_B64"]) @@ -434,9 +514,9 @@ class DeploymentInstallerTests(unittest.TestCase): ) self.assertEqual( "managed", - json.loads( - (root / "installation.json").read_text(encoding="utf-8") - )["components"]["postgres"]["mode"], + json.loads((root / "installation.json").read_text(encoding="utf-8"))[ + "components" + ]["postgres"]["mode"], ) self.assertEqual( 1, @@ -451,6 +531,42 @@ class DeploymentInstallerTests(unittest.TestCase): )[0], ) + def test_cli_requires_explicit_external_s3_values_after_garage(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + root = Path(directory) / "installation" + self.assertEqual( + 0, + run_cli( + [ + "init", + "--non-interactive", + "--directory", + str(root), + "--storage", + "garage", + ] + )[0], + ) + + result, _stdout, stderr = run_cli( + [ + "configure", + "--directory", + str(root), + "--storage", + "s3", + ] + ) + + self.assertEqual(1, result) + self.assertIn("switching to external S3 requires", stderr) + self.assertEqual( + "garage", + json.loads((root / "installation.json").read_text(encoding="utf-8"))[ + "components" + ]["storage"]["mode"], + ) + def test_deployer_builds_and_runs_as_one_zipapp(self) -> None: with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: output = Path(directory) / "govoplan-deploy.pyz" @@ -616,9 +732,7 @@ class DeploymentInstallerTests(unittest.TestCase): "http://127.0.0.1:8080/health", timeout_seconds=120.0, ) - receipt = json.loads( - (root / "receipt.json").read_text(encoding="utf-8") - ) + receipt = json.loads((root / "receipt.json").read_text(encoding="utf-8")) self.assertEqual("govoplan-local", receipt["installation_id"]) self.assertEqual( {"address": "127.0.0.1", "port": 8080}, @@ -639,6 +753,30 @@ class DeploymentInstallerTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "symbolic link"): bundle_paths(link) + def test_legacy_receipt_requests_direct_web_port_handoff(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-deploy-test-") as directory: + receipt = Path(directory) / "receipt.json" + receipt.write_text( + json.dumps({"services": ["api", "web", "worker"]}), + encoding="utf-8", + ) + self.assertTrue(_receipt_uses_direct_web_port(receipt)) + + receipt.write_text( + json.dumps( + { + "services": [ + "api", + "web", + "load-balancer", + "worker", + ] + } + ), + encoding="utf-8", + ) + self.assertFalse(_receipt_uses_direct_web_port(receipt)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_release_plan_guidance.py b/tests/test_release_plan_guidance.py index 18eb9e9..8ba6083 100644 --- a/tests/test_release_plan_guidance.py +++ b/tests/test_release_plan_guidance.py @@ -149,6 +149,28 @@ class ReleasePlanGuidanceTests(unittest.TestCase): self.assertIn('pill("recommended next", recommendationKind)', webui) self.assertIn("Remediation:", webui) + def test_webui_projects_release_state_into_a_guided_workflow(self) -> None: + webui = (RELEASE_ROOT / "webui" / "index.html").read_text(encoding="utf-8") + + for phase in ( + "Inspect", + "Targets", + "Validate", + "Source", + "Package", + "Publish", + "Verify", + ): + self.assertIn(f'label: "{phase}"', webui) + self.assertIn("releaseWorkflowPhases", webui) + self.assertIn("workflowPrimaryAction", webui) + self.assertIn("data-run-step-id", webui) + self.assertIn("invalidateReleaseDraft", webui) + self.assertIn("Installation Verification", webui) + self.assertIn("external release-integration CI gate", webui) + self.assertIn('const inspectionNotices = (summary.missing_count || 0)', webui) + self.assertIn('(summary.repository_count || 0) === 0', webui) + def dashboard(*, workspace: Path, version: str) -> ReleaseDashboard: repo = RepositorySnapshot( diff --git a/tools/deployment/govoplan_deploy/bundle.py b/tools/deployment/govoplan_deploy/bundle.py index 60f5d72..0557f07 100644 --- a/tools/deployment/govoplan_deploy/bundle.py +++ b/tools/deployment/govoplan_deploy/bundle.py @@ -20,6 +20,8 @@ from .model import ENV_NAME_PATTERN, InstallationSpec SPEC_FILENAME = "installation.json" ENV_FILENAME = "secrets.env" COMPOSE_FILENAME = "compose.json" +GARAGE_CONFIG_FILENAME = "garage.toml" +LOAD_BALANCER_CONFIG_FILENAME = "load-balancer.cfg" PLAN_FILENAME = "plan.json" RECEIPT_FILENAME = "receipt.json" LOCK_FILENAME = ".deployment.lock" @@ -52,6 +54,7 @@ RUNTIME_ENV_KEYS = ( "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET", + "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", ) @@ -61,6 +64,8 @@ class BundlePaths: spec: Path env: Path compose: Path + garage_config: Path + load_balancer_config: Path plan: Path receipt: Path lock: Path @@ -69,15 +74,15 @@ class BundlePaths: def bundle_paths(root: Path) -> BundlePaths: expanded = root.expanduser() if expanded.is_symlink(): - raise ValueError( - f"installation root must not be a symbolic link: {expanded}" - ) + raise ValueError(f"installation root must not be a symbolic link: {expanded}") resolved = expanded.resolve() return BundlePaths( root=resolved, spec=resolved / SPEC_FILENAME, env=resolved / ENV_FILENAME, compose=resolved / COMPOSE_FILENAME, + garage_config=resolved / GARAGE_CONFIG_FILENAME, + load_balancer_config=resolved / LOAD_BALANCER_CONFIG_FILENAME, plan=resolved / PLAN_FILENAME, receipt=resolved / RECEIPT_FILENAME, lock=resolved / LOCK_FILENAME, @@ -190,10 +195,37 @@ def reconcile_runtime_environment( values["GOVOPLAN_ALLOW_PROCESS_LOCAL_LOGIN_THROTTLE"] = "false" storage = spec.components.storage - values["FILE_STORAGE_BACKEND"] = storage.mode if storage.mode == "local": + values["FILE_STORAGE_BACKEND"] = "local" + values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false" values["FILE_STORAGE_LOCAL_ROOT"] = "/var/lib/govoplan/files" + elif storage.mode == "garage": + access_key = values.setdefault( + "GARAGE_DEFAULT_ACCESS_KEY", + f"GK{secrets.token_hex(16)}", + ) + secret_key = values.setdefault( + "GARAGE_DEFAULT_SECRET_KEY", + secrets.token_hex(32), + ) + bucket = values.setdefault("GARAGE_DEFAULT_BUCKET", "govoplan-files") + values.setdefault("GARAGE_RPC_SECRET", secrets.token_hex(32)) + values.setdefault("GARAGE_ADMIN_TOKEN", secrets.token_urlsafe(48)) + values.setdefault("GARAGE_METRICS_TOKEN", secrets.token_urlsafe(48)) + values.update( + { + "FILE_STORAGE_BACKEND": "s3", + "FILE_STORAGE_S3_ENDPOINT_URL": "http://garage:3900", + "FILE_STORAGE_S3_REGION": "garage", + "FILE_STORAGE_S3_ACCESS_KEY_ID": access_key, + "FILE_STORAGE_S3_SECRET_ACCESS_KEY": secret_key, + "FILE_STORAGE_S3_BUCKET": bucket, + "FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "true", + } + ) else: + values["FILE_STORAGE_BACKEND"] = "s3" + values["FILE_STORAGE_S3_DEPLOYMENT_MANAGED"] = "false" required = ( "FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", @@ -278,6 +310,50 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: } dependency_conditions["redis"] = {"condition": "service_healthy"} + if spec.components.storage.mode == "garage": + garage_environment = _environment_references( + ( + "GARAGE_DEFAULT_ACCESS_KEY", + "GARAGE_DEFAULT_SECRET_KEY", + "GARAGE_DEFAULT_BUCKET", + "GARAGE_RPC_SECRET", + "GARAGE_ADMIN_TOKEN", + "GARAGE_METRICS_TOKEN", + ), + required=True, + ) + services["garage"] = { + "image": spec.components.storage.image, + "restart": "unless-stopped", + "command": [ + "/garage", + "server", + "--single-node", + "--default-bucket", + ], + "environment": garage_environment, + "healthcheck": { + "test": ["CMD", "/garage", "status"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "15s", + }, + "labels": { + "org.govoplan.configuration-sha256": sha256( + render_garage_config().encode("utf-8") + ).hexdigest() + }, + "security_opt": ["no-new-privileges:true"], + "volumes": [ + f"./{GARAGE_CONFIG_FILENAME}:/etc/garage.toml:ro", + "garage-meta:/var/lib/garage/meta", + "garage-data:/var/lib/garage/data", + ], + "networks": ["internal"], + } + dependency_conditions["garage"] = {"condition": "service_healthy"} + if spec.components.mail.mode == "test-mail": services["test-mail"] = { "image": spec.components.mail.image, @@ -312,6 +388,7 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: services["api"] = { **common_runtime, "image": spec.release.api_image, + "scale": spec.replicas.api, "command": [ "python", "-m", @@ -342,15 +419,43 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: services["web"] = { "image": spec.release.web_image, "restart": "unless-stopped", - "environment": {"GOVOPLAN_API_UPSTREAM": "http://api:8000"}, - "depends_on": {"api": {"condition": "service_healthy"}}, + "scale": spec.replicas.web, + "environment": {"GOVOPLAN_API_UPSTREAM": "http://load-balancer:8000"}, + "networks": ["internal"], + } + services["load-balancer"] = { + "image": spec.components.load_balancer.image, + "restart": "unless-stopped", + "healthcheck": { + "test": [ + "CMD", + "haproxy", + "-c", + "-f", + "/usr/local/etc/haproxy/haproxy.cfg", + ], + "interval": "10s", + "timeout": "5s", + "retries": 10, + }, + "labels": { + "org.govoplan.configuration-sha256": sha256( + render_load_balancer_config(spec).encode("utf-8") + ).hexdigest() + }, "ports": [_published_port(spec.listen.address, spec.listen.port, 8080)], + "read_only": True, + "security_opt": ["no-new-privileges:true"], + "volumes": [ + (f"./{LOAD_BALANCER_CONFIG_FILENAME}:/usr/local/etc/haproxy/haproxy.cfg:ro") + ], "networks": ["internal"], } if spec.components.redis.mode != "disabled": services["worker"] = { **common_runtime, "image": spec.release.api_image, + "scale": spec.replicas.worker, "command": [ "python", "-m", @@ -388,6 +493,9 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: volumes["redis-data"] = {} if spec.components.storage.mode == "local": volumes["files-data"] = {} + if spec.components.storage.mode == "garage": + volumes["garage-meta"] = {} + volumes["garage-data"] = {} return { "name": spec.installation_id, @@ -402,14 +510,81 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]: } +def render_garage_config() -> str: + return """metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" + +[admin] +api_bind_addr = "[::]:3903" +""" + + +def render_load_balancer_config(spec: InstallationSpec) -> str: + return f"""global + log stdout format raw local0 + maxconn 4096 + +defaults + log global + mode http + option httplog + option redispatch + timeout connect 5s + timeout client 60s + timeout server 60s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold other 10s + hold refused 10s + hold nx 10s + hold timeout 10s + hold valid 10s + hold obsolete 10s + +frontend public_web + bind :8080 + default_backend web_replicas + +backend web_replicas + balance roundrobin + option httpchk GET /health + http-check expect status 200 + server-template web- {spec.replicas.web} web:8080 check resolvers docker init-addr libc,none + +frontend internal_api + bind :8000 + default_backend api_replicas + +backend api_replicas + balance leastconn + option httpchk GET /health + http-check expect status 200 + server-template api- {spec.replicas.api} api:8000 check resolvers docker init-addr libc,none +""" + + def service_names(spec: InstallationSpec) -> tuple[str, ...]: return tuple(render_compose(spec)["services"].keys()) def canonical_json(value: object) -> bytes: return ( - json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) - + "\n" + json.dumps(value, indent=2, sort_keys=True, separators=(",", ": ")) + "\n" ).encode("utf-8") @@ -439,9 +614,7 @@ def read_env(path: Path) -> dict[str, str]: if not separator or not ENV_NAME_PATTERN.fullmatch(key): raise ValueError(f"invalid environment line {line_number} in {path}") if key in values: - raise ValueError( - f"duplicate environment key {key!r} on line {line_number}" - ) + raise ValueError(f"duplicate environment key {key!r} on line {line_number}") value = raw_value if value.startswith('"'): try: @@ -464,9 +637,7 @@ def read_env(path: Path) -> dict[str, str]: def write_env(path: Path, values: Mapping[str, str]) -> None: invalid = sorted(key for key in values if not ENV_NAME_PATTERN.fullmatch(key)) if invalid: - raise ValueError( - "invalid environment variable names: " + ", ".join(invalid) - ) + raise ValueError("invalid environment variable names: " + ", ".join(invalid)) lines = [ "# Generated by govoplan-deploy. Keep this file private.", *[f"{key}={_env_value(value)}" for key, value in sorted(values.items())], @@ -477,9 +648,7 @@ def write_env(path: Path, values: Mapping[str, str]) -> None: def atomic_write(path: Path, payload: bytes, *, mode: int) -> None: path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - temporary = path.with_name( - f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp" - ) + temporary = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(8)}.tmp") descriptor = os.open( temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, @@ -506,8 +675,7 @@ def _env_value(value: str) -> str: if "\x00" in value or "\n" in value or "\r" in value: raise ValueError("environment values must not contain NUL or line breaks") if value and all( - character.isalnum() or character in "_./:@%+,-" - for character in value + character.isalnum() or character in "_./:@%+,-" for character in value ): return value escaped = value.replace("\\", "\\\\").replace("'", "\\'") @@ -550,9 +718,7 @@ def _single_quoted_env_value(value: str, *, line_number: int) -> str: continue index += 1 if index >= len(body) or body[index] not in {"\\", "'"}: - raise ValueError( - f"invalid single-quoted escape on line {line_number}" - ) + raise ValueError(f"invalid single-quoted escape on line {line_number}") output.append(body[index]) index += 1 return "".join(output) diff --git a/tools/deployment/govoplan_deploy/cli.py b/tools/deployment/govoplan_deploy/cli.py index 9af05d4..6fce6ad 100644 --- a/tools/deployment/govoplan_deploy/cli.py +++ b/tools/deployment/govoplan_deploy/cli.py @@ -30,13 +30,18 @@ from .bundle import ( read_env, reconcile_runtime_environment, render_compose, + render_garage_config, + render_load_balancer_config, service_names, write_env, ) from .model import ( ComponentConfig, + DEFAULT_GARAGE_IMAGE, + DEFAULT_LOAD_BALANCER_IMAGE, InstallationSpec, ListenConfig, + ReplicaConfig, SpecError, default_spec, load_spec, @@ -117,7 +122,12 @@ def _directory_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--directory", type=Path, - default=Path.home() / ".local" / "share" / "govoplan" / "installations" / "default", + default=Path.home() + / ".local" + / "share" + / "govoplan" + / "installations" + / "default", help="Private installation state directory.", ) @@ -140,7 +150,9 @@ def _configuration_arguments( choices=("managed", "external"), default=default("managed"), ) - parser.add_argument("--database-url", help="External PostgreSQL URL; stored privately.") + parser.add_argument( + "--database-url", help="External PostgreSQL URL; stored privately." + ) parser.add_argument( "--redis", choices=("managed", "external", "disabled"), @@ -154,17 +166,47 @@ def _configuration_arguments( ) parser.add_argument( "--storage", - choices=("local", "s3"), + choices=("local", "garage", "s3"), default=default("local"), ) + parser.add_argument( + "--garage-image", + default=default(DEFAULT_GARAGE_IMAGE), + help="Garage image used by managed object storage.", + ) parser.add_argument("--s3-endpoint-url", help="S3-compatible endpoint URL.") parser.add_argument("--s3-region", help="S3 region.") - parser.add_argument("--s3-access-key-id", help="S3 access key id; stored privately.") + parser.add_argument( + "--s3-access-key-id", help="S3 access key id; stored privately." + ) parser.add_argument( "--s3-secret-access-key", help="S3 secret access key; stored privately.", ) parser.add_argument("--s3-bucket", help="S3 bucket for managed files.") + parser.add_argument( + "--load-balancer-image", + default=default(DEFAULT_LOAD_BALANCER_IMAGE), + help="HAProxy image used by the managed local load balancer.", + ) + parser.add_argument( + "--api-replicas", + type=int, + default=default(1), + help="API containers on this Compose host (1-64).", + ) + parser.add_argument( + "--web-replicas", + type=int, + default=default(1), + help="WebUI containers on this Compose host (1-64).", + ) + parser.add_argument( + "--worker-replicas", + type=int, + default=None, + help="Worker containers on this Compose host (1-128, or 0 without Redis).", + ) parser.add_argument( "--module-set", choices=("core", "base", "full"), @@ -216,6 +258,11 @@ def _init(args: argparse.Namespace) -> int: redis_mode=args.redis, mail_mode=args.mail, storage_mode=args.storage, + garage_image=args.garage_image, + load_balancer_image=args.load_balancer_image, + api_replicas=args.api_replicas, + web_replicas=args.web_replicas, + worker_replicas=args.worker_replicas, module_set=args.module_set, api_image=args.api_image, web_image=args.web_image, @@ -260,9 +307,23 @@ def _configure(args: argparse.Namespace) -> int: and spec.components.redis.mode == "external" and not args.redis_url ): - raise ValueError( - "switching Redis to external requires --redis-url" - ) + raise ValueError("switching Redis to external requires --redis-url") + if current.components.storage.mode != "s3" and spec.components.storage.mode == "s3": + missing_s3_options = [ + flag + for flag, value in ( + ("--s3-endpoint-url", args.s3_endpoint_url), + ("--s3-region", args.s3_region), + ("--s3-access-key-id", args.s3_access_key_id), + ("--s3-secret-access-key", args.s3_secret_access_key), + ("--s3-bucket", args.s3_bucket), + ) + if not value + ] + if missing_s3_options: + raise ValueError( + "switching to external S3 requires: " + ", ".join(missing_s3_options) + ) secrets = read_env(paths.env) secrets.update(_supplied_secret_values(args)) secrets = reconcile_runtime_environment(spec, secrets) @@ -318,6 +379,8 @@ def _apply(args: argparse.Namespace) -> int: "components.postgres.image.", "components.redis.image.", "components.mail.image.", + "components.storage.image.", + "components.load_balancer.image.", "release.manifest", "modules.image_composition", ) @@ -344,15 +407,17 @@ def _apply(args: argparse.Namespace) -> int: _run([*compose, "pull"], cwd=paths.root) dependencies = [ name - for name in ("postgres", "redis", "test-mail") + for name in ("postgres", "redis", "garage", "test-mail") if name in service_names(spec) ] if dependencies: _run([*compose, "up", "--detach", *dependencies], cwd=paths.root) _run([*compose, "run", "--rm", "migrate"], cwd=paths.root) + if _receipt_uses_direct_web_port(paths.receipt): + _run([*compose, "stop", "web"], cwd=paths.root) runtime_services = [ name - for name in ("api", "web", "worker", "scheduler") + for name in ("api", "web", "load-balancer", "worker", "scheduler") if name in service_names(spec) ] _run( @@ -378,6 +443,11 @@ def _apply(args: argparse.Namespace) -> int: "web_image": spec.release.web_image, }, "services": list(service_names(spec)), + "replicas": { + "api": spec.replicas.api, + "web": spec.replicas.web, + "worker": spec.replicas.worker, + }, "listen": { "address": spec.listen.address, "port": spec.listen.port, @@ -470,6 +540,7 @@ def _updated_spec( api_image=args.api_image or current.release.api_image, web_image=args.web_image or current.release.web_image, ) + storage_mode = args.storage or current.components.storage.mode components = ComponentConfig( postgres=replace( current.components.postgres, @@ -485,7 +556,35 @@ def _updated_spec( ), storage=replace( current.components.storage, - mode=args.storage or current.components.storage.mode, + mode=storage_mode, + image=( + args.garage_image + or current.components.storage.image + or DEFAULT_GARAGE_IMAGE + ) + if storage_mode == "garage" + else "", + ), + load_balancer=replace( + current.components.load_balancer, + image=(args.load_balancer_image or current.components.load_balancer.image), + ), + ) + replicas = ReplicaConfig( + api=( + args.api_replicas if args.api_replicas is not None else current.replicas.api + ), + web=( + args.web_replicas if args.web_replicas is not None else current.replicas.web + ), + worker=( + args.worker_replicas + if args.worker_replicas is not None + else ( + 0 + if components.redis.mode == "disabled" + else max(current.replicas.worker, 1) + ) ), ) value = replace( @@ -499,6 +598,7 @@ def _updated_spec( ), release=release, components=components, + replicas=replicas, enabled_modules=modules, ) return parse_spec(value.to_dict()) @@ -514,9 +614,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]: s3_values = { "FILE_STORAGE_S3_ENDPOINT_URL": getattr(args, "s3_endpoint_url", None), "FILE_STORAGE_S3_REGION": getattr(args, "s3_region", None), - "FILE_STORAGE_S3_ACCESS_KEY_ID": getattr( - args, "s3_access_key_id", None - ), + "FILE_STORAGE_S3_ACCESS_KEY_ID": getattr(args, "s3_access_key_id", None), "FILE_STORAGE_S3_SECRET_ACCESS_KEY": getattr( args, "s3_secret_access_key", None ), @@ -528,9 +626,7 @@ def _supplied_secret_values(args: argparse.Namespace) -> dict[str, str]: def _pgtools_url(database_url: str) -> str: if database_url.startswith("postgresql+psycopg://"): - return "postgresql://" + database_url.removeprefix( - "postgresql+psycopg://" - ) + return "postgresql://" + database_url.removeprefix("postgresql+psycopg://") return database_url @@ -543,6 +639,16 @@ def _write_bundle( atomic_write(paths.spec, canonical_json(spec.to_dict()), mode=0o600) write_env(paths.env, secrets) atomic_write(paths.compose, canonical_json(render_compose(spec)), mode=0o600) + atomic_write( + paths.load_balancer_config, + render_load_balancer_config(spec).encode("utf-8"), + mode=0o644, + ) + atomic_write( + paths.garage_config, + render_garage_config().encode("utf-8"), + mode=0o644, + ) def _write_plan(path: Path, plan: DeploymentPlan) -> None: @@ -570,9 +676,7 @@ def _prompt_configuration(args: argparse.Namespace) -> None: if args.profile == "self-hosted" and args.public_url.startswith("http://"): args.public_url = "https://govoplan.example.org" args.public_url = _prompt("Public URL", args.public_url) - args.postgres = _prompt_choice( - "PostgreSQL", args.postgres, ("managed", "external") - ) + args.postgres = _prompt_choice("PostgreSQL", args.postgres, ("managed", "external")) if args.postgres == "external" and not args.database_url: args.database_url = getpass.getpass( "External PostgreSQL URL (input hidden): " @@ -582,13 +686,9 @@ def _prompt_configuration(args: argparse.Namespace) -> None: if args.profile == "self-hosted" else ("managed", "external", "disabled") ) - args.redis = _prompt_choice( - "Redis", args.redis, redis_choices - ) + args.redis = _prompt_choice("Redis", args.redis, redis_choices) if args.redis == "external" and not args.redis_url: - args.redis_url = getpass.getpass( - "External Redis URL (input hidden): " - ).strip() + args.redis_url = getpass.getpass("External Redis URL (input hidden): ").strip() mail_choices = ( ("disabled", "external-relay") if args.profile == "self-hosted" @@ -599,20 +699,43 @@ def _prompt_configuration(args: argparse.Namespace) -> None: args.mail, mail_choices, ) - args.storage = _prompt_choice("File storage", args.storage, ("local", "s3")) + args.storage = _prompt_choice( + "File storage", + args.storage, + ("local", "garage", "s3"), + ) if args.storage == "s3": args.s3_endpoint_url = args.s3_endpoint_url or _prompt( "S3 endpoint URL", "https://s3.example.org" ) args.s3_region = args.s3_region or _prompt("S3 region", "eu-central-1") - args.s3_access_key_id = args.s3_access_key_id or _prompt( - "S3 access key id", "" - ) + args.s3_access_key_id = args.s3_access_key_id or _prompt("S3 access key id", "") args.s3_secret_access_key = ( args.s3_secret_access_key or getpass.getpass("S3 secret access key (input hidden): ").strip() ) args.s3_bucket = args.s3_bucket or _prompt("S3 bucket", "govoplan-files") + args.api_replicas = _prompt_integer( + "API replicas on this host", + args.api_replicas, + minimum=1, + maximum=64, + ) + args.web_replicas = _prompt_integer( + "WebUI replicas on this host", + args.web_replicas, + minimum=1, + maximum=64, + ) + if args.redis != "disabled": + args.worker_replicas = _prompt_integer( + "Worker replicas on this host", + args.worker_replicas or 1, + minimum=1, + maximum=128, + ) + else: + args.worker_replicas = 0 args.module_set = _prompt_choice( "Initial module set", args.module_set, ("core", "base", "full") ) @@ -631,6 +754,24 @@ def _prompt_choice(label: str, default: str, choices: tuple[str, ...]) -> str: print(f"Choose one of: {', '.join(choices)}") +def _prompt_integer( + label: str, + default: int, + *, + minimum: int, + maximum: int, +) -> int: + while True: + raw = _prompt(label, str(default)) + try: + value = int(raw) + except ValueError: + value = minimum - 1 + if minimum <= value <= maximum: + return value + print(f"Choose a number between {minimum} and {maximum}.") + + @contextmanager def _deployment_lock(path: Path) -> Iterator[None]: descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) @@ -638,7 +779,9 @@ def _deployment_lock(path: Path) -> Iterator[None]: try: fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: - raise ValueError("another deployment operation owns the installation lock") from exc + raise ValueError( + "another deployment operation owns the installation lock" + ) from exc os.ftruncate(descriptor, 0) os.write(descriptor, f"{os.getpid()}\n".encode("ascii")) os.fsync(descriptor) @@ -686,7 +829,22 @@ def _parse_compose_ps(output: str) -> list[object]: return value if isinstance(value, list) else [value] +def _receipt_uses_direct_web_port(path: Path) -> bool: + if not path.exists(): + return False + try: + receipt = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + services = receipt.get("services") if isinstance(receipt, dict) else None + return ( + isinstance(services, list) + and "web" in services + and "load-balancer" not in services + ) + + def _now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace( - "+00:00", "Z" + return ( + datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") ) diff --git a/tools/deployment/govoplan_deploy/model.py b/tools/deployment/govoplan_deploy/model.py index 07bdfee..8d96033 100644 --- a/tools/deployment/govoplan_deploy/model.py +++ b/tools/deployment/govoplan_deploy/model.py @@ -12,6 +12,8 @@ from urllib.parse import urlsplit SCHEMA_VERSION = 1 +DEFAULT_GARAGE_IMAGE = "dxflrs/garage:v2.3.0" +DEFAULT_LOAD_BALANCER_IMAGE = "haproxy:3.2.21-alpine" INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$") ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$") SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") @@ -87,6 +89,7 @@ class ServiceConfig: @dataclass(frozen=True, slots=True) class StorageConfig: mode: str + image: str = "" @dataclass(frozen=True, slots=True) @@ -95,6 +98,14 @@ class ComponentConfig: redis: ServiceConfig mail: ServiceConfig storage: StorageConfig + load_balancer: ServiceConfig + + +@dataclass(frozen=True, slots=True) +class ReplicaConfig: + api: int + web: int + worker: int @dataclass(frozen=True, slots=True) @@ -107,6 +118,7 @@ class InstallationSpec: network_subnet: str release: ReleaseConfig components: ComponentConfig + replicas: ReplicaConfig enabled_modules: tuple[str, ...] def to_dict(self) -> dict[str, Any]: @@ -126,6 +138,11 @@ def default_spec( redis_mode: str = "managed", mail_mode: str = "disabled", storage_mode: str = "local", + garage_image: str = DEFAULT_GARAGE_IMAGE, + load_balancer_image: str = DEFAULT_LOAD_BALANCER_IMAGE, + api_replicas: int = 1, + web_replicas: int = 1, + worker_replicas: int | None = None, module_set: str = "base", api_image: str = "govoplan-api:unpublished", web_image: str = "govoplan-web:unpublished", @@ -141,6 +158,11 @@ def default_spec( }.get(module_set) if modules is None: raise SpecError("module_set must be core, base, or full") + effective_worker_replicas = ( + (0 if redis_mode == "disabled" else 1) + if worker_replicas is None + else worker_replicas + ) subnet_octet = 32 + (sum(installation_id.encode("utf-8")) % 192) raw = { "schema_version": SCHEMA_VERSION, @@ -173,7 +195,20 @@ def default_spec( "image": "greenmail/standalone:2.1.9", "url_env": "", }, - "storage": {"mode": storage_mode}, + "storage": { + "mode": storage_mode, + "image": garage_image if storage_mode == "garage" else "", + }, + "load_balancer": { + "mode": "managed", + "image": load_balancer_image, + "url_env": "", + }, + }, + "replicas": { + "api": api_replicas, + "web": web_replicas, + "worker": effective_worker_replicas, }, "enabled_modules": list(modules), } @@ -203,6 +238,7 @@ def parse_spec(raw: object) -> InstallationSpec: "network_subnet", "release", "components", + "replicas", "enabled_modules", }, "installation", @@ -233,11 +269,10 @@ def parse_spec(raw: object) -> InstallationSpec: port=_port(_integer(listen_raw, "port"), "listen.port"), ) - network_subnet = _network( - _string(root, "network_subnet"), "network_subnet" - ) + network_subnet = _network(_string(root, "network_subnet"), "network_subnet") release = _release(root.get("release")) components = _components(root.get("components"), profile=profile) + replicas = _replicas(root.get("replicas"), components=components) enabled_raw = root.get("enabled_modules") if not isinstance(enabled_raw, list): @@ -248,9 +283,7 @@ def parse_spec(raw: object) -> InstallationSpec: if not isinstance(value, str) or not re.fullmatch( r"[a-z][a-z0-9_]{1,63}", value ): - raise SpecError( - f"enabled_modules[{index}] must be a canonical module id" - ) + raise SpecError(f"enabled_modules[{index}] must be a canonical module id") if value in seen_modules: raise SpecError(f"enabled_modules contains duplicate module {value!r}") enabled_modules.append(value) @@ -265,6 +298,7 @@ def parse_spec(raw: object) -> InstallationSpec: network_subnet=network_subnet, release=release, components=components, + replicas=replicas, enabled_modules=tuple(enabled_modules), ) @@ -296,7 +330,9 @@ def _release(raw: object) -> ReleaseConfig: raise SpecError("release.manifest_url must use HTTPS") manifest_sha256 = _optional_string(value, "manifest_sha256").lower() if manifest_sha256 and not SHA256_PATTERN.fullmatch(manifest_sha256): - raise SpecError("release.manifest_sha256 must be a lowercase SHA-256 hex digest") + raise SpecError( + "release.manifest_sha256 must be a lowercase SHA-256 hex digest" + ) api_image = _image(_string(value, "api_image"), "release.api_image") web_image = _image(_string(value, "web_image"), "release.web_image") return ReleaseConfig( @@ -311,7 +347,11 @@ def _release(raw: object) -> ReleaseConfig: def _components(raw: object, *, profile: str) -> ComponentConfig: value = _mapping(raw, "components") - _only_keys(value, {"postgres", "redis", "mail", "storage"}, "components") + _only_keys( + value, + {"postgres", "redis", "mail", "storage", "load_balancer"}, + "components", + ) postgres = _service( value.get("postgres"), "components.postgres", @@ -334,14 +374,41 @@ def _components(raw: object, *, profile: str) -> ComponentConfig: url_env_required_for=set(), ) storage_raw = _mapping(value.get("storage"), "components.storage") - _only_keys(storage_raw, {"mode"}, "components.storage") - storage = StorageConfig(mode=_choice(storage_raw, "mode", {"local", "s3"})) + _only_keys(storage_raw, {"mode", "image"}, "components.storage") + storage_mode = _choice(storage_raw, "mode", {"local", "s3", "garage"}) + storage_image = _optional_string(storage_raw, "image") + if storage_mode == "garage": + storage_image = _image( + storage_image or DEFAULT_GARAGE_IMAGE, + "components.storage.image", + ) + elif storage_image: + raise SpecError( + "components.storage.image is only valid for managed Garage storage" + ) + storage = StorageConfig(mode=storage_mode, image=storage_image) + load_balancer = _service( + value.get( + "load_balancer", + { + "mode": "managed", + "image": DEFAULT_LOAD_BALANCER_IMAGE, + "url_env": "", + }, + ), + "components.load_balancer", + modes={"managed"}, + image_required_for={"managed"}, + url_env_required_for=set(), + ) if postgres.url_env != "DATABASE_URL": raise SpecError("components.postgres.url_env must be DATABASE_URL") if redis.url_env != "REDIS_URL": raise SpecError("components.redis.url_env must be REDIS_URL") if mail.url_env: raise SpecError("components.mail.url_env must be empty") + if load_balancer.url_env: + raise SpecError("components.load_balancer.url_env must be empty") if profile == "self-hosted" and redis.mode == "disabled": raise SpecError("self-hosted installations require managed or external Redis") if profile == "self-hosted" and mail.mode == "test-mail": @@ -351,9 +418,36 @@ def _components(raw: object, *, profile: str) -> ComponentConfig: redis=redis, mail=mail, storage=storage, + load_balancer=load_balancer, ) +def _replicas(raw: object, *, components: ComponentConfig) -> ReplicaConfig: + if raw is None: + return ReplicaConfig( + api=1, + web=1, + worker=0 if components.redis.mode == "disabled" else 1, + ) + value = _mapping(raw, "replicas") + _only_keys(value, {"api", "web", "worker"}, "replicas") + replicas = ReplicaConfig( + api=_bounded_integer(value, "api", minimum=1, maximum=64), + web=_bounded_integer(value, "web", minimum=1, maximum=64), + worker=_bounded_integer(value, "worker", minimum=0, maximum=128), + ) + if components.redis.mode == "disabled": + if replicas.worker: + raise SpecError("replicas.worker must be 0 when Redis is disabled") + if replicas.api > 1: + raise SpecError( + "multiple API replicas require Redis for shared throttling and queues" + ) + elif replicas.worker < 1: + raise SpecError("replicas.worker must be at least 1 when Redis is enabled") + return replicas + + def _service( raw: object, label: str, @@ -418,6 +512,19 @@ def _integer(value: Mapping[str, Any], key: str) -> int: return result +def _bounded_integer( + value: Mapping[str, Any], + key: str, + *, + minimum: int, + maximum: int, +) -> int: + result = _integer(value, key) + if result < minimum or result > maximum: + raise SpecError(f"{key} must be between {minimum} and {maximum}") + return result + + def _choice(value: Mapping[str, Any], key: str, choices: set[str]) -> str: result = _string(value, key) if result not in choices: @@ -434,9 +541,7 @@ def _http_url(value: str, label: str) -> str: if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise SpecError(f"{label} must be an absolute HTTP(S) URL") if parsed.username or parsed.password or parsed.fragment or parsed.query: - raise SpecError( - f"{label} must not contain credentials, a query, or a fragment" - ) + raise SpecError(f"{label} must not contain credentials, a query, or a fragment") if label == "public_url" and parsed.path not in {"", "/"}: raise SpecError("public_url must address the site root without a path") return value.rstrip("/") @@ -470,9 +575,7 @@ def _network(value: str, label: str) -> str: or not any(network.subnet_of(parent) for parent in RFC1918_NETWORKS) or not 24 <= network.prefixlen <= 28 ): - raise SpecError( - f"{label} must be an RFC1918 IPv4 /24 to /28 network" - ) + raise SpecError(f"{label} must be an RFC1918 IPv4 /24 to /28 network") return str(network) diff --git a/tools/deployment/govoplan_deploy/planning.py b/tools/deployment/govoplan_deploy/planning.py index 04cd2cc..744ee0b 100644 --- a/tools/deployment/govoplan_deploy/planning.py +++ b/tools/deployment/govoplan_deploy/planning.py @@ -68,9 +68,7 @@ class DeploymentPlan: "installation_id": self.installation_id, "desired_spec_sha256": self.desired_spec_sha256, "desired_compose_sha256": self.desired_compose_sha256, - "desired_environment_fingerprint": ( - self.desired_environment_fingerprint - ), + "desired_environment_fingerprint": (self.desired_environment_fingerprint), "blocked": self.blocked, "actions": [action.to_dict() for action in self.actions], "checks": [check.to_dict() for check in self.checks], @@ -151,7 +149,9 @@ def build_plan( and previous_services == desired_services ): actions.append( - PlanAction("noop", "installation", "Desired state matches the last receipt.") + PlanAction( + "noop", "installation", "Desired state matches the last receipt." + ) ) checks = list(static_checks(spec, paths)) @@ -179,6 +179,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. images["components.redis.image"] = spec.components.redis.image if spec.components.mail.mode == "test-mail": images["components.mail.image"] = spec.components.mail.image + if spec.components.storage.mode == "garage": + images["components.storage.image"] = spec.components.storage.image + images["components.load_balancer.image"] = spec.components.load_balancer.image for label, image in images.items(): if image_is_unpublished(image): @@ -248,7 +251,7 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. required = {"MASTER_KEY_B64", "DATABASE_URL"} if spec.components.redis.mode != "disabled": required.add("REDIS_URL") - if spec.components.storage.mode == "s3": + if spec.components.storage.mode in {"s3", "garage"}: required.update( { "FILE_STORAGE_S3_ENDPOINT_URL", @@ -258,6 +261,17 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. "FILE_STORAGE_S3_BUCKET", } ) + if spec.components.storage.mode == "garage": + required.update( + { + "GARAGE_DEFAULT_ACCESS_KEY", + "GARAGE_DEFAULT_SECRET_KEY", + "GARAGE_DEFAULT_BUCKET", + "GARAGE_RPC_SECRET", + "GARAGE_ADMIN_TOKEN", + "GARAGE_METRICS_TOKEN", + } + ) missing = sorted(name for name in required if not values.get(name)) checks.append( Check( @@ -268,7 +282,9 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. if missing else "Required runtime secret references are populated." ), - "Re-run configure with the required external service values." if missing else "", + "Re-run configure with the required external service values." + if missing + else "", ) ) if paths.env.exists(): @@ -312,6 +328,39 @@ def static_checks(spec: InstallationSpec, paths: BundlePaths) -> tuple[Check, .. "Include files-data in backup/restore drills or configure S3 storage.", ) ) + if spec.components.storage.mode == "garage": + checks.append( + Check( + "storage.garage.single_node", + "warning", + ( + "Managed Garage is persistent S3-compatible storage, but " + "this Compose profile runs one Garage node without data redundancy." + ), + ( + "Use an external multi-node Garage/S3 service and tested " + "backup/restore for an availability-sensitive installation." + ), + ) + ) + checks.append( + Check( + "topology.load_balancing", + "ok", + ( + "Managed HAProxy balances " + f"{spec.replicas.web} WebUI and {spec.replicas.api} API replica(s)." + ), + ) + ) + if spec.replicas.worker > 1: + checks.append( + Check( + "topology.worker_scaling", + "ok", + f"{spec.replicas.worker} workers share the configured Redis queues.", + ) + ) return tuple(checks) @@ -566,9 +615,7 @@ def _endpoint_check( ) -def _run_command( - argv: Sequence[str], cwd: Path -) -> subprocess.CompletedProcess[str]: +def _run_command(argv: Sequence[str], cwd: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( list(argv), cwd=cwd, diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html index ce1a019..5594fd2 100644 --- a/tools/release/webui/index.html +++ b/tools/release/webui/index.html @@ -60,6 +60,7 @@ .toolbar, .summary, + .release-guide, .layout { width: min(1600px, 100%); margin: 0 auto 18px; @@ -401,8 +402,146 @@ .workflow-stages { display: grid; - gap: 10px; + grid-template-columns: repeat(7, minmax(112px, 1fr)); + gap: 0; + padding: 18px 14px 14px; + overflow-x: auto; + } + + .workflow-stage { + position: relative; + min-width: 112px; + min-height: 86px; + border: 0; + border-radius: 0; + background: transparent; + color: var(--muted); + padding: 0 8px; + cursor: pointer; + } + + .workflow-stage::after { + content: ""; + position: absolute; + z-index: 0; + top: 17px; + left: calc(50% + 21px); + width: calc(100% - 42px); + height: 2px; + background: var(--line); + } + + .workflow-stage:last-child::after { + display: none; + } + + .workflow-stage.is-complete::after { + background: var(--ok); + } + + .workflow-stage:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; + } + + .workflow-stage-marker { + position: relative; + z-index: 1; + display: grid; + place-items: center; + width: 36px; + height: 36px; + margin: 0 auto 8px; + border: 2px solid var(--line); + border-radius: 50%; + background: var(--panel); + color: var(--muted); + font-size: 12px; + font-weight: 800; + } + + .workflow-stage.is-complete .workflow-stage-marker { + border-color: var(--ok); + background: var(--ok); + color: #fff; + } + + .workflow-stage.is-current .workflow-stage-marker { + border-color: var(--accent); + color: var(--accent); + box-shadow: 0 0 0 3px rgba(18, 97, 166, 0.12); + } + + .workflow-stage.is-blocked .workflow-stage-marker { + border-color: var(--danger); + color: var(--danger); + } + + .workflow-stage.is-unavailable { + cursor: not-allowed; + opacity: 0.72; + } + + .workflow-stage-label, + .workflow-stage-status { + display: block; + text-align: center; + } + + .workflow-stage-label { + color: var(--text); + font-size: 12px; + font-weight: 800; + } + + .workflow-stage-status { + margin-top: 3px; + color: var(--muted); + font-size: 11px; + font-weight: 600; + } + + .workflow-guidance { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 18px; + align-items: center; padding: 14px; + border-top: 1px solid var(--line); + background: var(--panel-alt); + } + + .workflow-guidance h3 { + margin: 0 0 5px; + font-size: 14px; + } + + .workflow-guidance p { + margin: 0; + color: var(--muted); + font-size: 12px; + line-height: 1.45; + } + + .workflow-guidance .action-meta { + margin-top: 5px; + } + + .workflow-guidance button { + min-width: 160px; + } + + .release-run-toolbar { + display: grid; + grid-template-columns: minmax(240px, 1fr) auto; + gap: 12px; + align-items: end; + padding: 14px; + border-bottom: 1px solid var(--line); + } + + .release-run-toolbar .button-row { + margin: 0; } .action { @@ -520,6 +659,15 @@ .insight-grid { grid-template-columns: 1fr; } + + .workflow-stages { + grid-template-columns: repeat(7, minmax(128px, 1fr)); + } + + .workflow-guidance, + .release-run-toolbar { + grid-template-columns: 1fr; + } } @@ -547,9 +695,24 @@
+
+
+

Release Workflow

+ Loading... +
+
+
+
+

Inspect the release workspace

+

The console is collecting repository, contract, migration, and publication state.

+
+ +
+
+
-
+

Release Control

@@ -558,7 +721,7 @@
-
+

Published Modules

@@ -581,7 +744,7 @@
-
+

Compatibility

@@ -602,7 +765,7 @@
-
+

Repositories

@@ -634,46 +797,33 @@
-
-
-
+
-

Durable Run State

+

Durable Release Run

no run selected
-
-

Durable execution. A run freezes the selected inputs and plan, claims each supported executor before its effect, and keeps explicit confirmation and reconciliation boundaries. Unsupported dependency-ordering or version-metadata steps stay visible and disabled.

-
-
- - -
-
- -
+
+
+ +
-
-
-
Create a run after selecting exact repository versions, or choose a saved run.
+
-
- -
-
-

Release Workflow

- +
+
Durable execution. Durable Run State preserves the frozen plan and mutation receipts across refreshes. Select exact repository versions, validate the plan, then create or resume a run.
-
+
-
+
+

1. Git Sync

@@ -700,7 +850,7 @@
-
+

2. Prepare Changes

@@ -725,7 +875,7 @@
-
+

3. Source Release Tag

idle · plan idle @@ -752,7 +902,7 @@
-
+

4. Signed Website Catalog

@@ -782,7 +932,7 @@
-
+

Catalog State

@@ -790,13 +940,13 @@
-
+
-

5. Install Catalog

- planned +

5. Installation Verification

+ external gate
-

Future work: signed module archives and an installer/update catalog for deployments.

+

The signed catalog and Python artifacts are produced by the durable candidate step. End-to-end installation verification still runs in release integration CI and is not yet a bounded local-console executor.

@@ -851,6 +1001,8 @@ unitCount: document.getElementById("unitCount"), workflowStageStatus: document.getElementById("workflowStageStatus"), workflowStages: document.getElementById("workflowStages"), + workflowGuidance: document.getElementById("workflowGuidance"), + workflowPrimaryAction: document.getElementById("workflowPrimaryAction"), catalog: document.getElementById("catalog"), catalogStatus: document.getElementById("catalogStatus"), actions: document.getElementById("actions"), @@ -909,8 +1061,10 @@ const releaseState = { rows: {}, dashboard: null, + currentPlan: null, manualRepo: "", currentRun: null, + workflowAction: null, runs: [], runNextCursor: null, runStoreAvailable: true, @@ -945,9 +1099,16 @@ elements.cancelManualTarget.addEventListener("click", closeManualTarget); elements.applyManualTarget.addEventListener("click", applyManualTarget); elements.manualTargetInput.addEventListener("input", validateManualTarget); + elements.workflowPrimaryAction.addEventListener("click", runWorkflowPrimaryAction); elements.candidateDir.addEventListener("input", () => { if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); }); + for (const control of [elements.channel, elements.publicCatalog, elements.online, elements.migrations]) { + control.addEventListener("change", () => { + invalidateReleaseDraft(); + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); + }); + } function api(path) { const separator = path.indexOf("?"); @@ -1068,6 +1229,7 @@ if (selectedId && releaseState.runs.some((run) => run.run_id === selectedId)) { elements.releaseRun.value = selectedId; } + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } function renderReleaseRunStoreUnavailable(error) { @@ -1082,6 +1244,7 @@ elements.resumeReleaseRun.disabled = true; elements.runStatus.textContent = "unavailable"; elements.runOutput.innerHTML = `

${pill("unavailable", "block")} Durable run storage cannot be read

${escapeHtml(error.message)}

Dashboard and release previews remain available. Correct the private state-directory problem, then refresh.

`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } async function createReleaseRun() { @@ -1204,6 +1367,7 @@ elements.runStatus.textContent = "no run selected"; elements.resumeReleaseRun.disabled = true; elements.runOutput.innerHTML = `
Create a run after selecting exact repository versions, or choose a saved run.
`; + renderIdlePlan(); return; } elements.runStatus.textContent = "loading..."; @@ -1218,6 +1382,7 @@ elements.runStatus.textContent = "unavailable"; elements.resumeReleaseRun.disabled = true; elements.runOutput.innerHTML = `

${pill("unavailable", "block")} Run cannot be read

${escapeHtml(error.message)}

`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } } @@ -1265,7 +1430,7 @@
` : ""; - return `
+ return `

${escapeHtml(`${step.order}. ${step.title || step.id}`)}

${pill(step.state, kind)}

${escapeHtml(step.detail || "")}

${step.result_code ? `

Result code: ${escapeHtml(step.result_code)}

` : ""} @@ -1305,6 +1470,7 @@ submit.addEventListener("click", () => executeReleaseRunStep(controls.dataset.runExecuteStep, required, submit)); if (preview) preview.addEventListener("click", () => previewReleaseRunStep(controls.dataset.runExecuteStep, preview)); } + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } async function executeReleaseRunStep(stepId, confirmation, button) { @@ -1572,62 +1738,245 @@ function renderWorkflowStages(data) { const s = data.summary || {}; const selected = elements.releaseUnits ? elements.releaseUnits.querySelectorAll("[data-release-check]:checked").length : 0; - const gitBlockers = (s.missing_count || 0) + (s.safe_directory_count || 0); - const branchDrift = (s.ahead_count || 0) + (s.behind_count || 0); - const releaseBlockers = gitBlockers + (s.behind_count || 0) + (s.no_head_count || 0) + (s.dirty_count || 0) + (s.error_count || 0); - const catalog = data.catalog || {}; - const hasCandidate = Boolean(elements.candidateDir?.value.trim()); - const catalogPublished = catalog.catalog_matches_public === true && catalog.keyring_matches_public === true; - const gitStatus = gitBlockers ? "blocked" : branchDrift ? "sync" : "current"; - const tagStatus = releaseBlockers ? "blocked" : selected ? "plan ready" : "select repos"; - const signedStatus = hasCandidate ? "candidate" : "waiting"; - const catalogStatus = catalogPublished ? "published" : catalog.public_checked ? "review" : "local"; - const prepareStatus = s.dirty_count ? "dirty" : "clean"; - - elements.workflowStageStatus.textContent = data.summary?.status || "unknown"; - elements.workflowStages.innerHTML = [ - workflowStage( - "1. Sync local/remote Git", - gitStatus, - gitBlockers ? "block" : branchDrift ? "warn" : "ok", - `${s.ahead_count || 0} ahead / ${s.behind_count || 0} behind / ${s.dirty_count || 0} dirty repositories` - ), - workflowStage( - "2. Prepare changes", - prepareStatus, - s.dirty_count ? "warn" : "ok", - s.dirty_count - ? `${s.dirty_count} dirty repositories need explicit prepare commits.` - : "No dirty worktrees in the current dashboard." - ), - workflowStage( - "3. Publish source release tags", - tagStatus, - releaseBlockers ? "block" : selected ? "ok" : "warn", - selected - ? `${selected} selected; preview here, then create a durable run and execute its enabled tag/push steps.` - : "Select repositories, set target versions, then preview the source release tags." - ), - workflowStage( - "4. Sign and publish website catalog", - signedStatus, - hasCandidate ? "ok" : "warn", - hasCandidate ? `Preview candidate: ${elements.candidateDir.value.trim()}` : "After every selected source tag is remotely published, use a durable run to generate and publish its receipt-bound candidate." - ), - workflowStage( - "5. Maintain install catalog", - catalogStatus, - catalogPublished ? "ok" : "warn", - "Future: signed module archives for installer and module update workflows." - ), - ].join(""); + const plan = releaseState.currentRun?.immutable?.plan || releaseState.currentPlan; + const run = releaseState.currentRun; + const phases = releaseWorkflowPhases({ summary: s, selected, plan, run }); + const completed = phases.filter((phase) => phase.state === "complete").length; + const blocked = phases.find((phase) => phase.state === "blocked" || phase.state === "unavailable"); + const active = phases.find((phase) => phase.state === "current") || blocked; + elements.workflowStageStatus.textContent = `${completed}/${phases.length} complete${run ? ` · ${run.state?.status || "unknown"}` : ""}`; + elements.workflowStages.innerHTML = phases.map(workflowStage).join(""); + for (const stage of elements.workflowStages.querySelectorAll("[data-workflow-target]")) { + stage.addEventListener("click", () => scrollToReleaseTarget(stage.dataset.workflowTarget)); + } + renderWorkflowGuidance({ summary: s, selected, plan, run, active }); } - function workflowStage(title, status, kind, detail) { - return `
-

${escapeHtml(title)}

${pill(status, kind)}
-

${escapeHtml(detail)}

-
`; + function releaseWorkflowPhases({ summary, selected, plan, run }) { + const inspectionNotices = (summary.missing_count || 0) + + (summary.safe_directory_count || 0) + + (summary.error_count || 0); + const inspectionBlockers = (summary.repository_count || 0) === 0 ? 1 : 0; + const phases = [ + { + id: "inspect", + label: "Inspect", + target: "releaseControlSection", + state: inspectionBlockers ? "blocked" : "complete", + status: inspectionBlockers + ? "no repositories" + : inspectionNotices + ? `${inspectionNotices} notices` + : "ready", + }, + { + id: "select", + label: "Targets", + target: "releaseTargetsSection", + state: selected || run ? "complete" : inspectionBlockers ? "locked" : "current", + status: selected || run ? `${selected || Object.keys(run?.immutable?.input?.repo_versions || {}).length} selected` : "select", + }, + validationWorkflowPhase({ selected, plan, run, inspectionBlockers }), + runWorkflowPhase({ + id: "source", + label: "Source", + target: "releaseRunSection", + run, + plan, + match: (step) => /:(preflight|tag|push)$/.test(step.id || ""), + waitingStatus: "create run", + }), + runWorkflowPhase({ + id: "package", + label: "Package", + target: "releaseRunSection", + run, + plan, + match: (step) => ["catalog:initial-version-metadata", "catalog:selective-generator"].includes(step.id), + waitingStatus: "waiting", + }), + runWorkflowPhase({ + id: "publish", + label: "Publish", + target: "releaseRunSection", + run, + plan, + match: (step) => step.id === "catalog:validate-sign-publish", + waitingStatus: "waiting", + }), + { + id: "verify", + label: "Verify", + target: "installVerificationSection", + state: run?.state?.status === "completed" ? "unavailable" : "locked", + status: run?.state?.status === "completed" ? "CI gate" : "waiting", + }, + ]; + const firstOpen = phases.findIndex((phase) => ["current", "blocked", "unavailable"].includes(phase.state)); + if (firstOpen >= 0) { + for (let index = firstOpen + 1; index < phases.length; index += 1) { + if (phases[index].state !== "complete") phases[index] = { ...phases[index], state: "locked" }; + } + } + return phases; + } + + function validationWorkflowPhase({ selected, plan, run, inspectionBlockers }) { + if (inspectionBlockers) return { id: "validate", label: "Validate", target: "sourceReleaseSection", state: "locked", status: "waiting" }; + if (!selected && !run) return { id: "validate", label: "Validate", target: "sourceReleaseSection", state: "locked", status: "waiting" }; + if (!plan) return { id: "validate", label: "Validate", target: "sourceReleaseSection", state: "current", status: "build plan" }; + if (plan.status === "blocked") return { id: "validate", label: "Validate", target: "sourceReleaseSection", state: "blocked", status: "blocked" }; + return { id: "validate", label: "Validate", target: "sourceReleaseSection", state: "complete", status: "gates pass" }; + } + + function runWorkflowPhase({ id, label, target, run, plan, match, waitingStatus }) { + if (!plan || plan.status === "blocked") return { id, label, target, state: "locked", status: waitingStatus }; + if (!run) return { + id, + label, + target, + state: id === "source" ? "current" : "locked", + status: id === "source" ? "create run" : waitingStatus, + }; + const steps = (run.state?.steps || []).filter(match); + if (!steps.length) return { id, label, target, state: "unavailable", status: "not available" }; + const succeeded = steps.filter((step) => step.state === "succeeded").length; + if (succeeded === steps.length) return { id, label, target, state: "complete", status: `${succeeded}/${steps.length}` }; + if (steps.some((step) => ["failed", "interrupted"].includes(step.state))) { + return { id, label, target, state: "blocked", status: `${succeeded}/${steps.length}` }; + } + if (steps.some((step) => step.state === "running" || step.available)) { + return { id, label, target, state: "current", status: `${succeeded}/${steps.length}` }; + } + const next = steps.find((step) => step.state === "pending"); + if (next && next.executor?.available === false && next.status !== "planned") { + return { id, label, target, state: "unavailable", status: next.status || "unavailable" }; + } + return { id, label, target, state: "locked", status: `${succeeded}/${steps.length}` }; + } + + function workflowStage(phase, index) { + const unavailable = phase.state === "unavailable"; + const marker = phase.state === "complete" ? "✓" : String(index + 1); + return ``; + } + + function renderWorkflowGuidance({ summary, selected, plan, run, active }) { + const recommendation = run?.recommended_next || plan?.recommended_action || null; + let guidance; + if ((summary.repository_count || 0) === 0) { + guidance = { + title: "Restore the release workspace", + detail: "No registered repositories are available for release planning.", + remediation: "Review Release Control, restore the workspace or repository registry, then refresh.", + button: "Review blockers", + action: { type: "scroll", target: "releaseControlSection" }, + }; + } else if (!selected && !run) { + guidance = { + title: "Select release targets", + detail: "Choose one or more repositories and confirm each exact target version.", + remediation: "The release plan will include only the selected independently versioned repositories.", + button: "Choose targets", + action: { type: "scroll", target: "releaseTargetsSection" }, + }; + } else if (!plan) { + guidance = { + title: "Validate the selected release", + detail: "Build a dry-run plan to check versions, contracts, migrations, Git state, and release composition.", + remediation: "Planning is read-only and does not create tags or artifacts.", + button: "Build plan", + action: { type: "build-plan" }, + }; + } else if (plan.status === "blocked") { + guidance = { + title: recommendation?.title || "Resolve release gates", + detail: recommendation?.detail || "The current plan contains blockers.", + remediation: recommendation?.remediation || "Review the plan findings, correct the source state, then build a fresh plan.", + button: "Review plan", + action: { type: "scroll", target: "sourceReleaseSection" }, + }; + } else if (!run) { + guidance = releaseState.runStoreAvailable ? { + title: "Freeze a durable release run", + detail: `${selected} repository target${selected === 1 ? "" : "s"} passed the plan-visible gates.`, + remediation: "Creating the run binds the exact plan, source commits, registered remotes, and ordered mutation boundaries.", + button: "Create durable run", + action: { type: "create-run" }, + } : { + title: "Restore durable run storage", + detail: "The release plan is valid, but the console cannot persist a safe execution record.", + remediation: "Review the run-storage error and restore the private state directory before executing release mutations.", + button: "Review storage error", + action: { type: "scroll", target: "releaseRunSection" }, + }; + } else if (recommendation?.step_id) { + guidance = { + title: recommendation.title || "Continue the release run", + detail: recommendation.detail || "The next durable step is ready for review.", + remediation: recommendation.remediation || "Open the step, review its prerequisites, and use its narrowly scoped confirmation.", + button: ["retry_step", "reconcile_step"].includes(recommendation.id) ? "Resolve step" : "Open next step", + action: { type: "run-step", target: recommendation.step_id }, + }; + } else if (run.state?.status === "completed") { + guidance = { + title: "Run installation verification", + detail: "Every durable source, package, signing, and publication step has a recorded success receipt.", + remediation: "End-to-end installation verification is still an external release-integration CI gate.", + button: "Review verification gate", + action: { type: "scroll", target: "installVerificationSection" }, + }; + } else { + guidance = { + title: recommendation?.title || active?.label || "Review release state", + detail: recommendation?.detail || "The next release action is not currently executable.", + remediation: recommendation?.remediation || "Review the blocked or unavailable step and its prerequisites.", + button: "Review run", + action: { type: "scroll", target: "releaseRunSection" }, + }; + } + releaseState.workflowAction = guidance.action; + elements.workflowGuidance.innerHTML = `
+

${escapeHtml(guidance.title)}

+

${escapeHtml(guidance.detail)}

+

Next: ${escapeHtml(guidance.remediation)}

+
+ `; + elements.workflowPrimaryAction = document.getElementById("workflowPrimaryAction"); + elements.workflowPrimaryAction.addEventListener("click", runWorkflowPrimaryAction); + } + + function runWorkflowPrimaryAction() { + const action = releaseState.workflowAction; + if (!action) return; + if (action.type === "build-plan") { + void buildSelectedPlan(); + return; + } + if (action.type === "create-run") { + void createReleaseRun(); + return; + } + if (action.type === "run-step") { + const step = elements.runOutput.querySelector(`[data-run-step-id="${cssEscape(action.target)}"]`); + if (step) { + step.scrollIntoView({ behavior: "smooth", block: "center" }); + step.querySelector("input:not(:disabled), select:not(:disabled), button:not(:disabled)")?.focus(); + return; + } + scrollToReleaseTarget("releaseRunSection"); + return; + } + scrollToReleaseTarget(action.target); + } + + function scrollToReleaseTarget(targetId) { + const target = document.getElementById(targetId); + if (target) target.scrollIntoView({ behavior: "smooth", block: "start" }); } function renderReleaseIntelligence(data) { @@ -1723,6 +2072,7 @@ input.addEventListener("change", () => { const repo = input.dataset.repo; releaseState.rows[repo] = { ...releaseState.rows[repo], selected: input.checked }; + invalidateReleaseDraft(); updateUnitCount(); }); } @@ -1868,6 +2218,7 @@ } function resetTarget(repoName) { + invalidateReleaseDraft(); const row = { ...releaseState.rows[repoName] }; delete row.target; releaseState.rows[repoName] = row; @@ -1880,6 +2231,7 @@ } setRowTargetTag(repoName, target); elements.statusLine.textContent = `${repoName} target reset.`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } function currentRowTarget(repoName) { @@ -1887,6 +2239,7 @@ } function setRowTarget(repoName, value) { + invalidateReleaseDraft(); releaseState.rows[repoName] = { ...releaseState.rows[repoName], target: value }; const element = rowTargetElement(repoName); if (element) { @@ -1894,6 +2247,7 @@ element.textContent = value; } setRowTargetTag(repoName, value); + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } function rowTargetElement(repoName) { @@ -2003,6 +2357,7 @@ } function renderPlan(plan) { + releaseState.currentPlan = plan; elements.planStatus.textContent = plan.status; const units = plan.units || []; const compatibility = plan.compatibility || []; @@ -2011,6 +2366,7 @@ const recommendation = plan.recommended_action || null; if (!units.length && !compatibility.length && !steps.length && !recommendation) { elements.actions.innerHTML = `
No selective release candidates. Select repositories above to plan a release.
`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); return; } const recommendationKind = plan.status === "blocked" ? "block" : plan.source_preflight_ready ? "ok" : "warn"; @@ -2061,11 +2417,27 @@ ${compatibilityHtml} ${stepHtml} `; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } function renderIdlePlan() { + releaseState.currentPlan = null; elements.planStatus.textContent = "idle"; elements.actions.innerHTML = `
Select repositories above, adjust target versions, then build a plan.
`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); + } + + function invalidateReleaseDraft() { + releaseState.currentPlan = null; + if (releaseState.currentRun) { + releaseState.currentRun = null; + elements.releaseRun.value = ""; + elements.runStatus.textContent = "no run selected"; + elements.resumeReleaseRun.disabled = true; + elements.runOutput.innerHTML = `
The repository selection changed. Build a fresh plan before creating a durable run; saved runs remain available from the selector.
`; + } + elements.planStatus.textContent = "idle"; + elements.actions.innerHTML = `
The release inputs changed. Build a fresh plan before continuing.
`; } async function generateCandidate() { @@ -2100,16 +2472,20 @@ setBusy([elements.buildSelectedPlan], true); try { if (!Object.keys(selectedRepoVersions()).length) { + releaseState.currentPlan = null; elements.planStatus.textContent = "idle"; elements.actions.innerHTML = `

${pill("idle", "warn")} No repositories selected

Select one or more rows in the repository table first.

`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); return; } const plan = await api("/api/selective-plan"); renderPlan(plan); elements.planStatus.textContent = plan.status; } catch (error) { + releaseState.currentPlan = null; elements.planStatus.textContent = "error"; elements.actions.innerHTML = `

${pill("error", "block")} Plan failed

${escapeHtml(error.message)}

`; + if (releaseState.dashboard) renderWorkflowStages(releaseState.dashboard); } finally { setBusy([elements.buildSelectedPlan], false); } @@ -2117,6 +2493,7 @@ function selectChangedUnits() { if (!releaseState.dashboard) return; + invalidateReleaseDraft(); let count = 0; for (const repo of releaseState.dashboard.repositories) { const current = primaryVersion(repo); @@ -2134,6 +2511,7 @@ function selectUnpushedUnits() { if (!releaseState.dashboard) return; + invalidateReleaseDraft(); let count = 0; for (const repo of releaseState.dashboard.repositories) { const current = primaryVersion(repo); @@ -2151,6 +2529,7 @@ function selectUnreleasedUnits() { if (!releaseState.dashboard) return; + invalidateReleaseDraft(); for (const repo of releaseState.dashboard.repositories) { if (repo.spec.name === "govoplan") continue; const unreleased = !primaryVersion(repo); @@ -2164,11 +2543,11 @@ } function clearSelectedUnits() { + invalidateReleaseDraft(); for (const repo of Object.keys(releaseState.rows)) { releaseState.rows[repo] = { ...releaseState.rows[repo], selected: false }; } if (releaseState.dashboard) renderReleaseUnits(releaseState.dashboard); - renderIdlePlan(); } function selectedRepoNames() {