From f1fd143ef5dcfe7484ba743d1540da5b44fa97e5 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 31 Jul 2026 05:46:50 +0200 Subject: [PATCH] Complete guided release console workflow --- docs/RELEASE_CONSOLE.md | 74 +- tests/test_release_execution.py | 189 +++- tests/test_release_migration_audit.py | 32 + tests/test_release_plan_guidance.py | 121 ++- tests/test_release_version_metadata.py | 105 ++ tools/release/govoplan_release/model.py | 1 + .../govoplan_release/release_execution.py | 990 +++++++++++++++++- tools/release/govoplan_release/release_run.py | 9 +- .../govoplan_release/selective_planner.py | 405 ++++++- .../govoplan_release/version_metadata.py | 387 +++++++ tools/release/release-migration-audit.py | 41 +- tools/release/server/app.py | 66 +- tools/release/webui/index.html | 29 +- 13 files changed, 2332 insertions(+), 117 deletions(-) create mode 100644 tests/test_release_version_metadata.py create mode 100644 tools/release/govoplan_release/version_metadata.py diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index 5338d78..18ae26d 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -22,10 +22,17 @@ effects can be verified safely: - freeze a selective plan as a durable, resumable local release run - durably preflight a creation-time-bound repository, create its annotated tag, and publish its branch/tag pair atomically +- deterministically update recognized package/manifest version declarations and + commit only the receipt-bound metadata paths +- order selected module providers before consumers, create module tags before + Core, regenerate Core's selected WebUI release lock, and re-run alignment + before any remote push - build selected Python wheels and generate a private, signed, receipt-bound catalog candidate - publish that exact candidate through a verified website commit and immutable tag after explicit confirmation +- install selected candidate wheels into a private no-network/no-dependency + target and verify their installed metadata against the frozen plan Start it from the meta repository: @@ -79,21 +86,23 @@ from the saved-run selector. Problems in unselected repositories remain visible as workspace notices but do not lock an unrelated release; the selective plan is the authority for blockers in the selected repository set. -Installation verification remains an explicit unavailable phase after a -durable run completes. It is currently enforced by release-integration CI, -rather than being presented as a successful local-console step. This prevents -the guide from treating source publication as proof that the published package -can be installed and started. +Installation verification is an explicit durable step after catalog +publication. It verifies the exact candidate receipt, installs every selected +Python wheel into a temporary target with network and dependency resolution +disabled, and compares installed names and versions with the frozen plan. +Deployment startup, database upgrades, and module-combination smoke tests remain +release-integration CI gates; the console does not represent its local wheel +check as a production deployment. `Build Plan` also returns structured release-gate findings for each selected repository. The plan names the recommended next action and gives an explicit remediation for source-version, lockfile, Core WebUI composition, Git state, and -worktree findings. A target version that has not yet been applied consistently -to the selected source tree is therefore explained before the source-tag -preflight rather than appearing only as an error after the operator tries to -tag. `source_preflight_ready` means that the plan-visible source gates pass; the -non-mutating `Preview Tag + Publish` remains mandatory for remote, manifest, and -immutable-tag checks. +worktree findings. A target version that differs from internally consistent +source metadata becomes a bounded `UPDATE` step. Unsupported, missing, or +internally inconsistent declarations remain a blocker with exact remediation. +`source_preflight_ready` means that plan-visible source gates pass or have a +bounded deterministic mutation; the non-mutating `Preview Tag + Publish` +remains mandatory for remote, manifest, and immutable-tag checks. ## Durable release runs @@ -103,11 +112,12 @@ requires the plan to resolve exactly the requested repositories and target versions; the browser cannot submit or replace the plan snapshot. The input and plan are then immutable and covered by a canonical SHA-256 integrity digest. Every executable repository step also carries its creation-time full HEAD, -branch, worktree state, target tag, and a SHA-256 over both the fetch and push -URLs of `origin`. Both URLs must exactly equal the remote registered in -`repositories.json`; a changed HEAD, branch, worktree, remote, or push URL -requires a new run rather than silently retargeting the frozen compatibility -decision. +branch, target tag, a SHA-256 over both the fetch and push URLs of `origin`, and +a SHA-256 over bounded dirty-path names and bytes. Both URLs must exactly equal +the remote registered in `repositories.json`; a changed HEAD, branch, worktree, +remote, push URL, or metadata byte requires a new run or explicit +interrupted-step reconciliation rather than silently retargeting the frozen +compatibility decision. The complete record also has a checksum so a valid-looking manual edit to its mutable state fails closed. File permissions remain the authority boundary; these digests detect accidental or manual corruption, not an attacker who can @@ -195,24 +205,32 @@ retry. The UI keeps unavailable controls visible and disabled. Supported executors durably claim the step before invoking an effect. Exact attempt replays return the recorded outcome and never invoke the executor a -second time. Successful repository preflight, tag, and push steps persist a -bounded repository-state receipt. Tag reconciliation independently requires an +second time. Successful repository preflight, version, commit, Core-bundle, +tag, and push steps persist a bounded repository-state receipt. Version +reconciliation requires aligned declarations in the same commit; commit +reconciliation requires the expected single-parent release commit and only +recognized metadata paths. Tag reconciliation independently requires an annotated local tag at the frozen HEAD; push reconciliation additionally requires both the remote annotated tag object and remote branch to match. Catalog generation persists only its server-issued opaque candidate ID and canonical catalog SHA-256, then re-resolves and re-hashes that private candidate before publication. -The safe baseline is intentionally narrower than the complete dry-run plan. -Dirty worktrees and version changes still show commit/version steps, but those -steps have no durable executor because the frozen run does not bind the exact -proposed file content. A run selecting Core together with one or more modules -begins with a disabled dependency-ordering barrier: local module tags, Core -release-lock regeneration/commit, Core tagging, alignment verification, and -pushes need an explicit DAG executor before that composition can mutate. -Operators must prepare and review those changes outside the console and create -a new run from the resulting clean HEAD. The console never skips these steps or -claims an end-to-end release succeeded. +Repository capabilities are frozen into each plan unit (`python-package`, +`webui-package`, `module-manifest`, `database-migrations`, `documentation`, +`core-release-bundle`, and the universal `git-source`) and determine which +steps appear. Internally aligned version changes are rendered deterministically +from recognized TOML, JSON, lockfile, manifest, and package declarations. +Pre-existing dirty worktrees remain visible but have no commit executor; the +console never absorbs unrelated operator changes. + +For mixed releases, module interface providers are ordered before consumers +and Core is tagged last. The durable sequence creates and commits module +metadata, creates local module tags, updates Core's selected WebUI references, +regenerates the release lock against those local tags, commits/tags Core, and +runs a receipt-bound alignment gate before exposing any atomic branch/tag push. +A failed step stops later steps while preserving prior receipts for explicit +retry or reconciliation. The browser likewise retains the request identifier for an uncertain resume/retry/reconciliation response and replays it after reload. A successful diff --git a/tests/test_release_execution.py b/tests/test_release_execution.py index 7b0955a..a1462e4 100644 --- a/tests/test_release_execution.py +++ b/tests/test_release_execution.py @@ -9,6 +9,7 @@ import tempfile from types import SimpleNamespace import unittest from unittest.mock import patch +import zipfile META_ROOT = Path(__file__).resolve().parents[1] @@ -17,6 +18,7 @@ if str(RELEASE_ROOT) not in sys.path: sys.path.insert(0, str(RELEASE_ROOT)) from govoplan_release.model import RepositorySpec # noqa: E402 +from govoplan_release.artifact_identity import inspect_python_wheel # noqa: E402 from govoplan_release.git_state import sanitized_git_environment # noqa: E402 from govoplan_release.release_execution import ( # noqa: E402 ExecutorSpec, @@ -30,6 +32,7 @@ from govoplan_release.release_execution import ( # noqa: E402 require_trusted_release_runtime, verify_frozen_repository_tag_receipt, verify_repository_preflight_binding, + verify_candidate_installation, _clone_catalog_sources, _checkout_frozen_source, _flatpak_proxy_environment, @@ -489,20 +492,180 @@ class ReleaseExecutionTests(unittest.TestCase): expected_receipt={"kind": "repository_state"}, ) - def test_commit_step_has_no_durable_executor(self) -> None: - self.assertIsNone( - executor_spec( - { - "id": "govoplan-files:commit", - "status": "planned", - "mutating": True, - "repo": "govoplan-files", - "source_binding": { - "kind": "repository_state", - }, - } - ) + def test_commit_step_has_narrow_confirmation(self) -> None: + spec = executor_spec( + { + "id": "govoplan-files:commit", + "status": "planned", + "mutating": True, + "repo": "govoplan-files", + "source_binding": { + "kind": "repository_state", + }, + } ) + self.assertIsNotNone(spec) + self.assertEqual("commit", spec.kind) + self.assertEqual("COMMIT", spec.confirmation) + + def test_version_and_commit_executors_bind_only_release_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workspace = Path(temp_dir) + repository = workspace / "govoplan-files" + remote = workspace / "files.git" + _git(workspace, "init", "--bare", str(remote)) + _git(workspace, "init", "-b", "main", str(repository)) + _git(repository, "config", "user.name", "Release Test") + _git(repository, "config", "user.email", "release@example.test") + (repository / "pyproject.toml").write_text( + '[project]\nname = "govoplan-files"\nversion = "1.2.3"\n', + encoding="utf-8", + ) + _git(repository, "add", "pyproject.toml") + _git(repository, "commit", "-m", "Initial") + _git(repository, "remote", "add", "origin", str(remote)) + _git(repository, "push", "-u", "origin", "main") + repository_spec = _repository_spec(remote) + plan_step = { + "id": "govoplan-files:version", + "repo": "govoplan-files", + "source_binding": {"kind": "repository_state"}, + } + with ( + patch( + "govoplan_release.release_execution.load_repository_specs", + return_value=(repository_spec,), + ), + patch( + "govoplan_release.release_execution.require_trusted_release_runtime" + ), + ): + before = repository_state_receipt( + repo="govoplan-files", + target_tag="v1.2.4", + workspace_root=workspace, + ) + _result, updated = execute_repository_step( + spec=ExecutorSpec( + "version", + "UPDATE", + "version_metadata_updated", + True, + ), + plan_step=plan_step, + repo_versions={"govoplan-files": "1.2.4"}, + workspace_root=workspace, + remote="origin", + expected_receipt=before, + ) + self.assertFalse(updated["worktree_clean"]) + self.assertNotEqual( + before["worktree_sha256"], + updated["worktree_sha256"], + ) + + plan_step["id"] = "govoplan-files:commit" + _result, committed = execute_repository_step( + spec=ExecutorSpec( + "commit", + "COMMIT", + "release_commit_created", + True, + ), + plan_step=plan_step, + repo_versions={"govoplan-files": "1.2.4"}, + workspace_root=workspace, + remote="origin", + expected_receipt=updated, + ) + + self.assertTrue(committed["worktree_clean"]) + self.assertNotEqual(before["head"], committed["head"]) + self.assertIn( + 'version = "1.2.4"', + (repository / "pyproject.toml").read_text(encoding="utf-8"), + ) + + def test_candidate_installation_verifies_installed_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workspace = Path(temp_dir) + repository = workspace / "govoplan-files" + repository.mkdir() + (repository / "pyproject.toml").write_text( + '[project]\nname = "govoplan-files"\nversion = "1.2.4"\n', + encoding="utf-8", + ) + artifacts = workspace / "candidate" / "artifacts" + artifacts.mkdir(parents=True) + wheel = artifacts / "govoplan_files-1.2.4-py3-none-any.whl" + members = { + "govoplan_files/__init__.py": '__version__ = "1.2.4"\n', + "govoplan_files-1.2.4.dist-info/METADATA": ( + "Metadata-Version: 2.1\n" + "Name: govoplan-files\n" + "Version: 1.2.4\n" + ), + "govoplan_files-1.2.4.dist-info/WHEEL": ( + "Wheel-Version: 1.0\n" + "Generator: release-test\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n" + ), + } + record = "\n".join(f"{name},," for name in members) + record += "\ngovoplan_files-1.2.4.dist-info/RECORD,,\n" + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in members.items(): + archive.writestr(name, content) + archive.writestr( + "govoplan_files-1.2.4.dist-info/RECORD", + record, + ) + identity = inspect_python_wheel(wheel) + channels = workspace / "candidate" / "channels" + channels.mkdir() + (channels / "stable.json").write_text( + json.dumps( + { + "release": { + "selected_units": [ + { + "repo": "govoplan-files", + "version": "1.2.4", + } + ], + "artifacts": [ + { + "package_name": "govoplan-files", + "archive_sha256": identity.archive_sha256, + } + ], + }, + "core_release": {}, + "modules": [ + { + "python_package": "govoplan-files", + "python_ref": ( + "govoplan-files @ " + "git+ssh://git@example.test/GovOPlaN/" + "govoplan-files.git@v1.2.4" + ), + } + ], + } + ) + + "\n", + encoding="utf-8", + ) + + result = verify_candidate_installation( + candidate_path=workspace / "candidate", + repo_versions={"govoplan-files": "1.2.4"}, + ) + + self.assertEqual("verified", result["status"]) + self.assertEqual("govoplan-files", result["packages"][0]["repo"]) + self.assertEqual("1.2.4", result["packages"][0]["version"]) def test_frozen_checkout_uses_receipt_commit_not_live_worktree(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: diff --git a/tests/test_release_migration_audit.py b/tests/test_release_migration_audit.py index 958cdad..8da412f 100644 --- a/tests/test_release_migration_audit.py +++ b/tests/test_release_migration_audit.py @@ -47,6 +47,38 @@ branch_labels: Union[str, Sequence[str], None] = None self.assertEqual(migration.depends_on, ("core",)) self.assertEqual(migration.branch_labels, ()) + def test_development_wrapper_uses_release_revision_metadata(self) -> None: + audit = load_audit_module() + with tempfile.TemporaryDirectory(prefix="migration-audit-test-") as directory: + root = Path(directory) + versions = root / "versions" + development = root / "dev_versions" + versions.mkdir() + development.mkdir() + release = versions / "1234_example.py" + release.write_text( + 'revision = "1234"\n' + 'down_revision = "base"\n' + 'depends_on = "core"\n' + "branch_labels = None\n", + encoding="utf-8", + ) + wrapper = development / release.name + wrapper.write_text( + "revision = _migration.revision\n" + "down_revision = _migration.down_revision\n" + "depends_on = _migration.depends_on\n" + "branch_labels = _migration.branch_labels\n", + encoding="utf-8", + ) + + migration = audit.parse_migration_file("govoplan-core", wrapper) + + self.assertIsNotNone(migration) + self.assertEqual("1234", migration.revision) + self.assertEqual(("base",), migration.down_revisions) + self.assertEqual(("core",), migration.depends_on) + def test_release_baseline_matches_current_heads_in_strict_report(self) -> None: audit = load_audit_module() migrations = [ diff --git a/tests/test_release_plan_guidance.py b/tests/test_release_plan_guidance.py index 8ba6083..79716eb 100644 --- a/tests/test_release_plan_guidance.py +++ b/tests/test_release_plan_guidance.py @@ -23,7 +23,7 @@ from govoplan_release.selective_planner import build_selective_release_plan # n class ReleasePlanGuidanceTests(unittest.TestCase): - def test_unprepared_target_has_structured_remediation_and_recommendation( + def test_unprepared_target_gets_bounded_version_and_commit_steps( self, ) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -40,20 +40,25 @@ class ReleasePlanGuidanceTests(unittest.TestCase): repo_versions={"govoplan-files": "1.2.4"}, ) - self.assertEqual("blocked", plan.status) - self.assertFalse(plan.source_preflight_ready) - finding = next( - item - for item in plan.gate_findings - if item.code == "repository_version_alignment" + self.assertEqual("attention", plan.status) + self.assertTrue(plan.source_preflight_ready) + self.assertFalse( + any( + item.code == "repository_version_alignment" + for item in plan.gate_findings + ) ) - self.assertEqual("govoplan-files", finding.repo) - self.assertEqual("pyproject.toml", finding.source) - self.assertEqual("1.2.4", finding.expected) - self.assertEqual("1.2.3", finding.actual) - self.assertIn("Regenerate lockfiles", finding.remediation) - self.assertEqual("resolve_release_gate", plan.recommended_action.id) - self.assertEqual("govoplan-files", plan.recommended_action.repo) + steps = {step.id: step for step in plan.dry_run_steps} + self.assertEqual("planned", steps["govoplan-files:version"].status) + self.assertEqual("planned", steps["govoplan-files:commit"].status) + self.assertIn("pyproject.toml", steps["govoplan-files:version"].detail) + self.assertTrue( + any( + "version metadata will be updated" in warning + for warning in plan.units[0].warnings + ) + ) + self.assertIn("python-package", plan.units[0].capabilities) def test_aligned_target_recommends_non_mutating_tag_preview(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: @@ -74,6 +79,91 @@ class ReleasePlanGuidanceTests(unittest.TestCase): self.assertEqual("preview_source_release", plan.recommended_action.id) self.assertIn("Preview Tag + Publish", plan.recommended_action.remediation) + def test_mixed_plan_tags_modules_before_core_and_aligns_before_push( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workspace = Path(temp_dir) + for repo_name in ("govoplan-core", "govoplan-files"): + repo_path = workspace / repo_name + repo_path.mkdir() + (repo_path / "pyproject.toml").write_text( + f'[project]\nname = "{repo_name}"\nversion = "1.2.3"\n', + encoding="utf-8", + ) + repositories = tuple( + RepositorySnapshot( + spec=RepositorySpec( + name=repo_name, + category="core" if repo_name == "govoplan-core" else "module", + subtype="platform" if repo_name == "govoplan-core" else "infrastructure", + remote=f"git@example.test:GovOPlaN/{repo_name}.git", + path=repo_name, + ), + absolute_path=str(workspace / repo_name), + exists=True, + is_git=True, + has_head=True, + branch="main", + versions=VersionSnapshot(pyproject="1.2.3"), + local_target_tag_exists=False, + ) + for repo_name in ("govoplan-core", "govoplan-files") + ) + base = dashboard(workspace=workspace, version="1.2.3") + mixed = ReleaseDashboard( + generated_at=base.generated_at, + meta_root=base.meta_root, + workspace_root=base.workspace_root, + target_version=None, + target_tag=None, + online=False, + include_migrations=False, + summary=DashboardSummary( + repository_count=2, + missing_count=0, + dirty_count=0, + ahead_count=0, + behind_count=0, + no_head_count=0, + error_count=0, + safe_directory_count=0, + local_target_tag_missing_count=0, + status="ready", + ), + repositories=repositories, + catalog=base.catalog, + ) + plan = build_selective_release_plan( + mixed, + selected_repos=("govoplan-core", "govoplan-files"), + repo_versions={ + "govoplan-core": "1.2.3", + "govoplan-files": "1.2.3", + }, + ) + + self.assertEqual( + ["govoplan-files", "govoplan-core"], + [unit.repo for unit in plan.units], + ) + step_ids = [step.id for step in plan.dry_run_steps] + self.assertLess( + step_ids.index("govoplan-files:tag"), + step_ids.index("govoplan-core:tag"), + ) + self.assertLess( + step_ids.index("govoplan-core:tag"), + step_ids.index("release:alignment"), + ) + self.assertLess( + step_ids.index("release:alignment"), + step_ids.index("govoplan-files:push"), + ) + self.assertEqual("release:install-verify", step_ids[-1]) + core = next(unit for unit in plan.units if unit.repo == "govoplan-core") + self.assertIn("core-release-bundle", core.capabilities) + def test_malformed_version_metadata_is_a_structured_blocker(self) -> None: cases = ( ("package.json", "{not-json\n", "JSONDecodeError"), @@ -167,7 +257,8 @@ class ReleasePlanGuidanceTests(unittest.TestCase): self.assertIn("data-run-step-id", webui) self.assertIn("invalidateReleaseDraft", webui) self.assertIn("Installation Verification", webui) - self.assertIn("external release-integration CI gate", webui) + self.assertIn("receipt-bound local gate", webui) + self.assertIn('step.id === "release:install-verify"', webui) self.assertIn('const inspectionNotices = (summary.missing_count || 0)', webui) self.assertIn('(summary.repository_count || 0) === 0', webui) diff --git a/tests/test_release_version_metadata.py b/tests/test_release_version_metadata.py new file mode 100644 index 0000000..fc6f3d9 --- /dev/null +++ b/tests/test_release_version_metadata.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile +import unittest + + +META_ROOT = Path(__file__).resolve().parents[1] +RELEASE_ROOT = META_ROOT / "tools" / "release" +if str(RELEASE_ROOT) not in sys.path: + sys.path.insert(0, str(RELEASE_ROOT)) + +from govoplan_release.version_metadata import ( # noqa: E402 + apply_version_metadata_mutations, + version_metadata_mutations, +) + + +class ReleaseVersionMetadataTests(unittest.TestCase): + def test_updates_recognized_metadata_without_changing_interface_versions( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + package = root / "src" / "govoplan_example" + backend = package / "backend" + webui = root / "webui" + backend.mkdir(parents=True) + webui.mkdir() + (root / "pyproject.toml").write_text( + '[project]\nname = "govoplan-example"\nversion = "1.2.3"\n', + encoding="utf-8", + ) + (root / "package.json").write_text( + '{"name":"root","version":"1.2.3"}\n', + encoding="utf-8", + ) + (webui / "package.json").write_text( + '{"name":"@govoplan/example-webui","version":"1.2.3"}\n', + encoding="utf-8", + ) + (webui / "package-lock.json").write_text( + json.dumps( + { + "name": "@govoplan/example-webui", + "version": "1.2.3", + "packages": { + "": { + "name": "@govoplan/example-webui", + "version": "1.2.3", + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + (backend / "manifest.py").write_text( + "def get_manifest():\n" + " return ModuleManifest(\n" + ' id="example",\n' + ' version="1.2.3",\n' + " provides_interfaces=(\n" + ' ModuleInterfaceProvider(name="example.api", version="4.0"),\n' + " ),\n" + " )\n", + encoding="utf-8", + ) + (package / "__init__.py").write_text( + '__version__ = "1.2.3"\n', + encoding="utf-8", + ) + + preview = version_metadata_mutations( + root, + target_version="1.2.4", + ) + changed = apply_version_metadata_mutations( + root, + target_version="1.2.4", + ) + + self.assertEqual( + {mutation.path for mutation in preview}, + set(changed), + ) + self.assertEqual( + "1.2.4", + json.loads((webui / "package-lock.json").read_text())[ + "packages" + ][""]["version"], + ) + manifest = (backend / "manifest.py").read_text(encoding="utf-8") + self.assertIn('version="1.2.4"', manifest) + self.assertIn('version="4.0"', manifest) + self.assertEqual( + '__version__ = "1.2.4"\n', + (package / "__init__.py").read_text(encoding="utf-8"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/govoplan_release/model.py b/tools/release/govoplan_release/model.py index 8d9b5cc..5e3bac4 100644 --- a/tools/release/govoplan_release/model.py +++ b/tools/release/govoplan_release/model.py @@ -222,6 +222,7 @@ class ReleasePlanUnit: status: str blockers: tuple[str, ...] = () warnings: tuple[str, ...] = () + capabilities: tuple[str, ...] = () provides_interfaces: tuple[InterfaceProviderSnapshot, ...] = () requires_interfaces: tuple[InterfaceRequirementSnapshot, ...] = () gate_findings: tuple[ReleaseGateFinding, ...] = () diff --git a/tools/release/govoplan_release/release_execution.py b/tools/release/govoplan_release/release_execution.py index 4bac04a..56852ca 100644 --- a/tools/release/govoplan_release/release_execution.py +++ b/tools/release/govoplan_release/release_execution.py @@ -4,6 +4,7 @@ from __future__ import annotations from dataclasses import dataclass import hashlib +import importlib.metadata import json import os from pathlib import Path @@ -12,11 +13,13 @@ import shutil import signal import stat import subprocess +import sys import tempfile import tomllib from typing import Any from .artifact_identity import ( + ArtifactIdentityError, MAX_WHEEL_BYTES, inspect_python_wheel, normalize_package_name, @@ -58,6 +61,14 @@ from .selective_catalog import ( trusted_keys_from_keyring, ) from .source_provenance import catalog_source_selection +from .version_alignment import ( + selected_release_webui_bundle_issues, + selected_repository_version_issues, +) +from .version_metadata import ( + VersionMetadataError, + apply_version_metadata_mutations, +) from .workspace import ( META_ROOT, REPOSITORIES_FILE, @@ -73,6 +84,8 @@ MAX_REMOTE_METADATA_BYTES = 16 * 1024 MAX_CATALOG_SOURCE_REPOSITORIES = 128 MAX_TRUSTED_RUNTIME_ENTRIES = 10_000 MAX_TRUSTED_GIT_ENTRIES = 500_000 +MAX_WORKTREE_RECEIPT_BYTES = 64 * 1024 * 1024 +MAX_WORKTREE_RECEIPT_PATHS = 2_048 DURABLE_REMOTE = "origin" FLATPAK_BUILDER_PROXY = Path("/usr/bin/flatpak-spawn") HOST_BUBBLEWRAP = Path("/usr/bin/bwrap") @@ -80,6 +93,9 @@ FLATPAK_HOST_BUS = Path("/run/flatpak/bus") _WEBUI_RELEASE_SOURCE = re.compile( r"/(?Pgovoplan-[a-z0-9-]+)\.git#v(?P[0-9]+\.[0-9]+\.[0-9]+)$" ) +_PYTHON_RELEASE_SOURCE = re.compile( + r"/(?Pgovoplan-[a-z0-9-]+)\.git@v(?P[^\s;]+)$" +) class ReleaseExecutionError(RuntimeError): @@ -104,8 +120,17 @@ class ExecutorSpec: EXECUTOR_SPECS = { "preflight": ExecutorSpec("preflight", "", "preflight_valid", False), + "version": ExecutorSpec("version", "UPDATE", "version_metadata_updated", True), + "commit": ExecutorSpec("commit", "COMMIT", "release_commit_created", True), + "bundle": ExecutorSpec("core_bundle", "LOCK", "release_bundle_locked", True), "tag": ExecutorSpec("tag", "TAG", "tag_created", True), "push": ExecutorSpec("push", "PUBLISH", "tag_published", True), + "release:alignment": ExecutorSpec( + "alignment", "", "release_alignment_valid", False + ), + "release:install-verify": ExecutorSpec( + "install_verify", "", "installation_verified", False + ), "catalog:selective-generator": ExecutorSpec( "catalog_generate", "GENERATE", "catalog_candidate_ready", True ), @@ -125,6 +150,13 @@ def executor_spec(plan_step: dict[str, Any]) -> ExecutorSpec | None: spec = None if spec is None or plan_step.get("status") != "planned": return None + if spec.kind in {"alignment", "install_verify"}: + return ( + spec + if plan_step.get("repo") is None + and plan_step.get("mutating") is False + else None + ) if plan_step.get("mutating") is not spec.mutating: return None if spec.kind == "catalog_generate": @@ -140,6 +172,8 @@ def executor_spec(plan_step: dict[str, Any]) -> ExecutorSpec | None: ) repo = plan_step.get("repo") binding = plan_step.get("source_binding") + if spec.kind == "core_bundle" and repo != "govoplan-core": + return None return ( spec if isinstance(repo, str) @@ -261,6 +295,7 @@ def execute_repository_step( workspace_root: Path, remote: str, expected_receipt: dict[str, Any] | None, + source_receipts: dict[str, dict[str, Any]] | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: require_trusted_release_runtime(workspace_root=workspace_root) if remote != DURABLE_REMOTE: @@ -290,6 +325,32 @@ def execute_repository_step( workspace_root=workspace_root, expected_receipt=expected_receipt, ) + if spec.kind == "version": + result, receipt = update_repository_version_metadata( + repo=repo, + version=version, + workspace_root=workspace_root, + expected_receipt=expected_receipt, + ) + return result, receipt + if spec.kind == "commit": + result, receipt = commit_repository_release_metadata( + repo=repo, + version=version, + repo_versions=repo_versions, + workspace_root=workspace_root, + expected_receipt=expected_receipt, + allow_core_bundle=repo == "govoplan-core", + ) + return result, receipt + if spec.kind == "core_bundle": + result, receipt = regenerate_core_release_bundle( + repo_versions=repo_versions, + workspace_root=workspace_root, + expected_receipt=expected_receipt, + source_receipts=source_receipts or {}, + ) + return result, receipt if spec.kind == "tag": result = tag_repositories( repos=(repo,), @@ -394,6 +455,302 @@ def repository_preflight( }, receipt +def update_repository_version_metadata( + *, + repo: str, + version: str, + workspace_root: Path, + expected_receipt: dict[str, Any] | None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Update only recognized version declarations in a clean frozen checkout.""" + + path = _repository_path(repo, workspace_root) + current = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_receipt_match(current, expected_receipt, operation="version update") + if not current["worktree_clean"]: + raise ReleaseExecutionBlocked( + "Version metadata update requires a clean frozen worktree." + ) + if current["tag_object"] is not None: + raise ReleaseExecutionBlocked( + "Version metadata cannot change after the target tag already exists." + ) + try: + changed = apply_version_metadata_mutations( + path, + target_version=version, + ) + except VersionMetadataError as exc: + raise ReleaseExecutionBlocked(str(exc)) from exc + if not changed: + raise ReleaseExecutionBlocked( + "Version executor found no metadata change for the frozen target." + ) + issues = selected_repository_version_issues( + repo_versions={repo: version}, + workspace=workspace_root, + ) + if issues: + raise ReleaseExecutionAmbiguous( + "Version metadata was written but post-update alignment failed; " + "reconcile or discard the bounded worktree changes." + ) + receipt = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + if receipt["worktree_clean"]: + raise ReleaseExecutionAmbiguous( + "Version metadata executor produced no verifiable worktree delta." + ) + return { + "status": "updated", + "repo": repo, + "version": version.removeprefix("v"), + "paths": list(changed), + }, receipt + + +def commit_repository_release_metadata( + *, + repo: str, + version: str, + repo_versions: dict[str, str], + workspace_root: Path, + expected_receipt: dict[str, Any] | None, + allow_core_bundle: bool, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Commit only the exact receipt-bound release metadata delta.""" + + path = _repository_path(repo, workspace_root) + current = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_receipt_match(current, expected_receipt, operation="release commit") + dirty_paths = _require_release_owned_dirty_paths( + path=path, + allow_core_bundle=allow_core_bundle, + ) + if not dirty_paths: + raise ReleaseExecutionBlocked( + "Release commit has no receipt-bound metadata changes." + ) + issues = selected_repository_version_issues( + repo_versions={repo: version}, + workspace=workspace_root, + ) + if issues: + raise ReleaseExecutionBlocked( + "Release metadata is not aligned to the frozen target version." + ) + if allow_core_bundle: + selected = _selected_webui_repo_versions( + repo_versions=repo_versions, + workspace_root=workspace_root, + ) + if selected and selected_release_webui_bundle_issues( + repo_versions={**selected, repo: version}, + workspace=workspace_root, + ): + raise ReleaseExecutionBlocked( + "Core release WebUI composition is not aligned." + ) + add_result = git(path, "add", "--", *dirty_paths, timeout=30) + if add_result.returncode != 0: + raise ReleaseExecutionBlocked("Could not stage bounded release metadata.") + staged = _git_changed_paths(path, cached=True) + if staged != dirty_paths: + raise ReleaseExecutionBlocked( + "Git staged paths differ from the receipt-bound release metadata." + ) + message = f"Release {repo} v{version.removeprefix('v')}" + commit_result = git(path, "commit", "-m", message, timeout=120) + if commit_result.returncode != 0: + raise ReleaseExecutionBlocked("Could not create the bounded release commit.") + receipt = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + if not receipt["worktree_clean"] or receipt["head"] == current["head"]: + raise ReleaseExecutionAmbiguous( + "Release commit returned success without a clean, new repository HEAD." + ) + return { + "status": "committed", + "repo": repo, + "head": receipt["head"], + "paths": list(dirty_paths), + }, receipt + + +def regenerate_core_release_bundle( + *, + repo_versions: dict[str, str], + workspace_root: Path, + expected_receipt: dict[str, Any] | None, + source_receipts: dict[str, dict[str, Any]], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Update Core WebUI refs and regenerate its release lock after module tags.""" + + core_version = repo_versions.get("govoplan-core") + if core_version is None: + raise ReleaseExecutionBlocked( + "Core bundle regeneration requires a selected Core release." + ) + core = _repository_path("govoplan-core", workspace_root) + current = repository_state_receipt( + repo="govoplan-core", + target_tag=f"v{core_version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_receipt_match(current, expected_receipt, operation="Core bundle update") + _require_release_owned_dirty_paths(path=core, allow_core_bundle=False) + selected = _selected_webui_repo_versions( + repo_versions=repo_versions, + workspace_root=workspace_root, + ) + for repo, version in selected.items(): + receipt = source_receipts.get(repo) + if not isinstance(receipt, dict): + raise ReleaseExecutionBlocked( + f"Core bundle requires the preceding tag receipt for {repo}." + ) + verified = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_receipt_match( + verified, + receipt, + operation=f"Core bundle source tag for {repo}", + ) + if verified.get("tag_object") is None: + raise ReleaseExecutionBlocked( + f"Core bundle requires an annotated local tag for {repo}." + ) + if verified.get("target_tag") != f"v{version.removeprefix('v')}": + raise ReleaseExecutionBlocked( + f"Core bundle receipt has the wrong tag for {repo}." + ) + + package_path = core / "webui" / "package.release.json" + lock_path = core / "webui" / "package-lock.release.json" + package_before = _read_regular_file(package_path, label="Core release package") + lock_before = _read_regular_file(lock_path, label="Core release lock") + try: + package = json.loads(package_before) + except json.JSONDecodeError as exc: + raise ReleaseExecutionBlocked("Core release package is malformed.") from exc + dependencies = package.get("dependencies") if isinstance(package, dict) else None + if not isinstance(dependencies, dict): + raise ReleaseExecutionBlocked( + "Core release package has no dependency object." + ) + updated_packages: list[str] = [] + for repo, version in selected.items(): + module_package = _read_json_object( + workspace_root / repo / "webui" / "package.json", + label=f"{repo} WebUI package", + ) + package_name = module_package.get("name") + if not isinstance(package_name, str) or package_name not in dependencies: + raise ReleaseExecutionBlocked( + f"Core release package has no dependency slot for {repo}." + ) + reference = dependencies[package_name] + if not isinstance(reference, str): + raise ReleaseExecutionBlocked( + f"Core release dependency for {repo} is not a Git reference." + ) + match = _WEBUI_RELEASE_SOURCE.search(reference) + if match is None or match.group("repo") != repo: + raise ReleaseExecutionBlocked( + f"Core release dependency for {repo} is not repository-bound." + ) + dependencies[package_name] = ( + reference[: match.start("version")] + + version.removeprefix("v") + + reference[match.end("version") :] + ) + updated_packages.append(package_name) + rendered_package = ( + json.dumps(package, indent=2, ensure_ascii=True, separators=(",", ": ")) + + "\n" + ).encode("utf-8") + _replace_regular_file(package_path, rendered_package) + command = [ + "/usr/bin/bash", + str(META_ROOT / "tools" / "release" / "generate-release-lock.sh"), + "--core-root", + str(core), + ] + for repo in selected: + command.extend(("--local-git-repo", str(workspace_root / repo))) + try: + result = subprocess.run( + tuple(command), + cwd=META_ROOT, + env=sanitized_git_environment(), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=300, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + _replace_regular_file(package_path, package_before) + _replace_regular_file(lock_path, lock_before) + raise ReleaseExecutionBlocked( + "Core release-lock generator could not be executed." + ) from exc + if result.returncode != 0: + _replace_regular_file(package_path, package_before) + _replace_regular_file(lock_path, lock_before) + raise ReleaseExecutionBlocked( + "Core release-lock regeneration failed: " + + _bounded_process_detail(result) + ) + issues = selected_release_webui_bundle_issues( + repo_versions=repo_versions, + workspace=workspace_root, + ) + version_issues = selected_repository_version_issues( + repo_versions={"govoplan-core": core_version}, + workspace=workspace_root, + ) + if issues or version_issues: + _replace_regular_file(package_path, package_before) + _replace_regular_file(lock_path, lock_before) + raise ReleaseExecutionBlocked( + "Regenerated Core release lock did not pass version/composition alignment." + ) + receipt = repository_state_receipt( + repo="govoplan-core", + target_tag=f"v{core_version.removeprefix('v')}", + workspace_root=workspace_root, + ) + if receipt["worktree_clean"]: + raise ReleaseExecutionBlocked( + "Core release bundle was already aligned; rebuild the release plan." + ) + return { + "status": "locked", + "repo": "govoplan-core", + "modules": sorted(selected), + "packages": sorted(updated_packages), + }, receipt + + def repository_state_receipt( *, repo: str, @@ -462,11 +819,511 @@ def repository_state_receipt( ).encode("utf-8") ).hexdigest(), "worktree_clean": not snapshot.dirty_entries, + "worktree_sha256": _worktree_sha256(path), "target_tag": target_tag, "tag_object": tag_object, } +def _worktree_sha256(path: Path) -> str: + """Bind dirty path names and bytes without persisting their contents.""" + + result = git(path, "status", "--porcelain=v1", "-z", "--untracked-files=all") + encoded = result.stdout.encode("utf-8", errors="strict") + if result.returncode != 0 or len(encoded) > 4 * 1024 * 1024: + raise ReleaseExecutionBlocked("Repository worktree state cannot be bound safely.") + records = [record for record in result.stdout.split("\0") if record] + paths: list[str] = [] + index = 0 + while index < len(records): + record = records[index] + if len(record) < 4 or record[2] != " ": + raise ReleaseExecutionBlocked( + "Repository worktree status has an unsupported shape." + ) + status = record[:2] + paths.append(record[3:]) + index += 1 + if "R" in status or "C" in status: + if index >= len(records): + raise ReleaseExecutionBlocked( + "Repository rename state cannot be bound safely." + ) + paths.append(records[index]) + index += 1 + canonical_paths = tuple(sorted(set(paths))) + if len(canonical_paths) > MAX_WORKTREE_RECEIPT_PATHS: + raise ReleaseExecutionBlocked( + "Repository worktree exceeds its receipt path bound." + ) + digest = hashlib.sha256() + digest.update(encoded) + observed_bytes = 0 + root = path.resolve() + for relative_name in canonical_paths: + relative = Path(relative_name) + if ( + relative.is_absolute() + or not relative.parts + or any(part in {"", ".", ".."} for part in relative.parts) + ): + raise ReleaseExecutionBlocked( + "Repository worktree contains a non-canonical dirty path." + ) + candidate = path / relative + try: + metadata = candidate.lstat() + except FileNotFoundError: + digest.update(b"\0deleted\0") + digest.update(relative_name.encode("utf-8")) + continue + except OSError as exc: + raise ReleaseExecutionBlocked( + "Repository worktree changed while its receipt was created." + ) from exc + if ( + candidate.resolve().parent != root + and root not in candidate.resolve().parents + ): + raise ReleaseExecutionBlocked( + "Repository worktree dirty path leaves the checkout." + ) + if stat.S_ISDIR(metadata.st_mode): + continue + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise ReleaseExecutionBlocked( + "Repository worktree dirty paths must be regular files." + ) + observed_bytes += metadata.st_size + if observed_bytes > MAX_WORKTREE_RECEIPT_BYTES: + raise ReleaseExecutionBlocked( + "Repository worktree exceeds its receipt byte bound." + ) + digest.update(b"\0file\0") + digest.update(relative_name.encode("utf-8")) + digest.update(b"\0") + try: + digest.update(candidate.read_bytes()) + except OSError as exc: + raise ReleaseExecutionBlocked( + "Repository worktree changed while its receipt was created." + ) from exc + return digest.hexdigest() + + +def _repository_path(repo: str, workspace_root: Path) -> Path: + specs = {item.name: item for item in load_repository_specs(include_website=False)} + repository = specs.get(repo) + if repository is None: + raise ReleaseExecutionBlocked("Repository is not registered.") + return resolve_repo_path(repository, workspace_root) + + +def _require_release_owned_dirty_paths( + *, + path: Path, + allow_core_bundle: bool, +) -> tuple[str, ...]: + del allow_core_bundle + dirty = _git_changed_paths(path, cached=False) + unsupported = tuple( + relative + for relative in dirty + if not _release_metadata_path(relative) + ) + if unsupported: + raise ReleaseExecutionBlocked( + "Release executor refuses non-metadata worktree paths: " + + ", ".join(unsupported[:8]) + ) + return dirty + + +def _release_metadata_path(relative: str) -> bool: + if relative in { + "pyproject.toml", + "package.json", + "package-lock.json", + "webui/package.json", + "webui/package-lock.json", + "webui/package.release.json", + "webui/package-lock.release.json", + }: + return True + parts = Path(relative).parts + return ( + len(parts) == 3 + and parts[0] == "src" + and parts[2] == "__init__.py" + ) or ( + len(parts) >= 4 + and parts[0] == "src" + and parts[-2:] == ("backend", "manifest.py") + ) + + +def _git_changed_paths(path: Path, *, cached: bool) -> tuple[str, ...]: + if cached: + result = git(path, "diff", "--cached", "--name-only", "-z", timeout=30) + if result.returncode != 0: + raise ReleaseExecutionBlocked("Staged release paths cannot be inspected.") + return tuple( + sorted({item for item in result.stdout.split("\0") if item}) + ) + result = git(path, "status", "--porcelain=v1", "-z", "--untracked-files=all") + if result.returncode != 0: + raise ReleaseExecutionBlocked("Release worktree paths cannot be inspected.") + records = [item for item in result.stdout.split("\0") if item] + paths: list[str] = [] + index = 0 + while index < len(records): + record = records[index] + if len(record) < 4 or record[2] != " ": + raise ReleaseExecutionBlocked( + "Release worktree status has an unsupported shape." + ) + status = record[:2] + paths.append(record[3:]) + index += 1 + if "R" in status or "C" in status: + if index >= len(records): + raise ReleaseExecutionBlocked( + "Release worktree rename state is incomplete." + ) + paths.append(records[index]) + index += 1 + return tuple(sorted(set(paths))) + + +def _selected_webui_repo_versions( + *, + repo_versions: dict[str, str], + workspace_root: Path, +) -> dict[str, str]: + return { + repo: version + for repo, version in sorted(repo_versions.items()) + if repo != "govoplan-core" + and (workspace_root / repo / "webui" / "package.json").is_file() + } + + +def _read_regular_file(path: Path, *, label: str) -> bytes: + try: + metadata = path.lstat() + except OSError as exc: + raise ReleaseExecutionBlocked(f"{label} is unavailable.") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_size > 16 * 1024 * 1024 + ): + raise ReleaseExecutionBlocked( + f"{label} must be a bounded regular file." + ) + try: + return path.read_bytes() + except OSError as exc: + raise ReleaseExecutionBlocked(f"{label} cannot be read.") from exc + + +def _read_json_object(path: Path, *, label: str) -> dict[str, Any]: + try: + value = json.loads(_read_regular_file(path, label=label)) + except json.JSONDecodeError as exc: + raise ReleaseExecutionBlocked(f"{label} is malformed.") from exc + if not isinstance(value, dict): + raise ReleaseExecutionBlocked(f"{label} must be an object.") + return value + + +def _replace_regular_file(path: Path, payload: bytes) -> None: + metadata = path.stat() + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.release-", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, metadata.st_mode & 0o777) + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + temporary.replace(path) + except Exception: + try: + os.close(descriptor) + except OSError: + pass + temporary.unlink(missing_ok=True) + raise + + +def _bounded_process_detail(result: subprocess.CompletedProcess[str]) -> str: + detail = (result.stderr or result.stdout or "unknown error").strip() + return detail[:MAX_EXECUTOR_OUTPUT] + + +def verify_release_alignment( + *, + repo_versions: dict[str, str], + workspace_root: Path, + source_receipts: dict[str, dict[str, Any]], +) -> dict[str, Any]: + """Recheck local tags and composition before the first remote mutation.""" + + verified: list[dict[str, str]] = [] + for repo, version in sorted(repo_versions.items()): + expected = source_receipts.get(repo) + current = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_receipt_match( + current, + expected, + operation=f"release alignment for {repo}", + ) + if current["tag_object"] is None or not current["worktree_clean"]: + raise ReleaseExecutionBlocked( + f"Release alignment requires a clean annotated local tag for {repo}." + ) + verified.append( + { + "repo": repo, + "version": version.removeprefix("v"), + "head": str(current["head"]), + "tag_object": str(current["tag_object"]), + } + ) + version_issues = selected_repository_version_issues( + repo_versions=repo_versions, + workspace=workspace_root, + ) + bundle_issues = selected_release_webui_bundle_issues( + repo_versions=repo_versions, + workspace=workspace_root, + ) + if version_issues or bundle_issues: + raise ReleaseExecutionBlocked( + "Post-tag release alignment reported " + f"{len(version_issues)} version and {len(bundle_issues)} " + "Core WebUI composition issue(s)." + ) + return { + "status": "aligned", + "repositories": verified, + } + + +def verify_candidate_installation( + *, + candidate_path: Path, + repo_versions: dict[str, str], +) -> dict[str, Any]: + """Install selected wheels without network/dependencies and verify metadata.""" + + artifacts = candidate_path / "artifacts" + try: + wheels = tuple(sorted(artifacts.glob("*.whl"))) + except OSError as exc: + raise ReleaseExecutionBlocked( + "Candidate wheel directory cannot be inspected." + ) from exc + expected, artifact_hashes = _candidate_python_expectations( + candidate_path=candidate_path, + repo_versions=repo_versions, + ) + try: + identities = [inspect_python_wheel(wheel) for wheel in wheels] + except ArtifactIdentityError as exc: + raise ReleaseExecutionBlocked("Candidate wheel identity is invalid.") from exc + observed = { + identity.package_name: identity.package_version for identity in identities + } + expected_versions = { + package: version for package, (_repo, version) in expected.items() + } + if observed != expected_versions: + raise ReleaseExecutionBlocked( + "Candidate wheels do not exactly match the selected Python package set." + ) + if any( + artifact_hashes.get(identity.package_name) != identity.archive_sha256 + for identity in identities + ): + raise ReleaseExecutionBlocked( + "Candidate wheel bytes do not match the signed catalog identities." + ) + if not expected_versions: + return {"status": "verified", "packages": []} + with tempfile.TemporaryDirectory(prefix="govoplan-install-verify-") as temp_dir: + root = Path(temp_dir) + target = root / "site" + target.mkdir(mode=0o700) + command = ( + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-index", + "--no-deps", + "--no-compile", + "--target", + str(target), + *(str(wheel) for wheel in wheels), + ) + environment = { + "PATH": "/usr/bin:/bin", + "HOME": str(root), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PYTHONNOUSERSITE": "1", + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_CACHE_DIR": "1", + "PIP_NO_INDEX": "1", + } + try: + result = subprocess.run( + command, + cwd=root, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=180, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ReleaseExecutionBlocked( + "Candidate installation verifier could not run." + ) from exc + if result.returncode != 0: + raise ReleaseExecutionBlocked( + "Candidate wheel installation failed: " + + _bounded_process_detail(result) + ) + installed = { + normalize_package_name(distribution.metadata["Name"]): distribution.version + for distribution in importlib.metadata.distributions(path=[str(target)]) + if distribution.metadata.get("Name") + } + if installed != expected_versions: + raise ReleaseExecutionBlocked( + "Installed candidate metadata does not match the frozen release plan." + ) + return { + "status": "verified", + "packages": [ + { + "repo": expected[identity.package_name][0], + "package": identity.package_name, + "version": identity.package_version, + "wheel_sha256": identity.archive_sha256, + } + for identity in sorted(identities, key=lambda item: item.package_name) + ], + } + + +def _candidate_python_expectations( + *, + candidate_path: Path, + repo_versions: dict[str, str], +) -> tuple[dict[str, tuple[str, str]], dict[str, str]]: + channel_dir = candidate_path / "channels" + catalogs = tuple(sorted(channel_dir.glob("*.json"))) + if len(catalogs) != 1: + raise ReleaseExecutionBlocked( + "Candidate must contain exactly one release channel catalog." + ) + catalog = _read_json_object(catalogs[0], label="candidate catalog") + release = catalog.get("release") + selected_units = release.get("selected_units") if isinstance(release, dict) else None + artifacts = release.get("artifacts") if isinstance(release, dict) else None + if not isinstance(selected_units, list) or not isinstance(artifacts, list): + raise ReleaseExecutionBlocked( + "Candidate catalog has no selected-unit artifact identity." + ) + selected_versions = { + str(item.get("repo")): str(item.get("version")) + for item in selected_units + if isinstance(item, dict) + and isinstance(item.get("repo"), str) + and isinstance(item.get("version"), str) + } + normalized_plan = { + repo: version.removeprefix("v") + for repo, version in sorted(repo_versions.items()) + } + if selected_versions != normalized_plan: + raise ReleaseExecutionBlocked( + "Candidate selected units do not match the frozen release plan." + ) + + entries: list[object] = [catalog.get("core_release")] + modules = catalog.get("modules") + if isinstance(modules, list): + entries.extend(modules) + expected: dict[str, tuple[str, str]] = {} + for entry in entries: + if not isinstance(entry, dict): + continue + reference = entry.get("python_ref") + package = entry.get("python_package") + match = ( + _PYTHON_RELEASE_SOURCE.search(reference) + if isinstance(reference, str) + else None + ) + if ( + match is None + or match.group("repo") not in normalized_plan + or not isinstance(package, str) + ): + continue + repo = match.group("repo") + version = normalized_plan[repo] + if match.group("version").removeprefix("v") != version: + raise ReleaseExecutionBlocked( + f"Candidate Python reference has the wrong version for {repo}." + ) + expected[normalize_package_name(package)] = (repo, version) + + artifact_hashes: dict[str, str] = {} + for artifact in artifacts: + if not isinstance(artifact, dict): + raise ReleaseExecutionBlocked( + "Candidate catalog contains a malformed artifact identity." + ) + package = artifact.get("package_name") + digest = artifact.get("archive_sha256") + if ( + not isinstance(package, str) + or not isinstance(digest, str) + or not re.fullmatch(r"[0-9a-f]{64}", digest) + ): + raise ReleaseExecutionBlocked( + "Candidate catalog contains a malformed artifact identity." + ) + normalized = normalize_package_name(package) + if normalized in artifact_hashes: + raise ReleaseExecutionBlocked( + "Candidate catalog contains duplicate artifact identities." + ) + artifact_hashes[normalized] = digest + if set(artifact_hashes) != set(expected): + raise ReleaseExecutionBlocked( + "Candidate artifact identities do not match selected Python packages." + ) + return expected, artifact_hashes + + def _require_operator_checkout(path: Path) -> None: _require_trusted_ancestor_chain(path) required = ( @@ -753,7 +1610,7 @@ def verify_repository_step_precondition( workspace_root: Path, expected_receipt: dict[str, Any] | None, ) -> dict[str, Any]: - if spec.kind not in {"tag", "push"}: + if spec.kind not in {"version", "commit", "core_bundle", "tag", "push"}: raise ReleaseExecutionBlocked("Repository precondition is unsupported.") if not isinstance(expected_receipt, dict): raise ReleaseExecutionBlocked( @@ -765,10 +1622,24 @@ def verify_repository_step_precondition( workspace_root=workspace_root, ) _require_receipt_match(current, expected_receipt, operation=spec.kind) - if not current["worktree_clean"]: + if spec.kind in {"version", "tag", "push"} and not current["worktree_clean"]: raise ReleaseExecutionBlocked( "Repository worktree changed after the run was frozen; create a new run." ) + if spec.kind == "core_bundle": + _require_release_owned_dirty_paths( + path=_repository_path(str(plan_step["repo"]), workspace_root), + allow_core_bundle=False, + ) + if spec.kind == "commit": + dirty_paths = _require_release_owned_dirty_paths( + path=_repository_path(str(plan_step["repo"]), workspace_root), + allow_core_bundle=str(plan_step["repo"]) == "govoplan-core", + ) + if not dirty_paths: + raise ReleaseExecutionBlocked( + "Release commit requires receipt-bound metadata changes." + ) if spec.kind == "push" and current["tag_object"] is None: raise ReleaseExecutionBlocked("Tag publication requires an annotated local tag.") return current @@ -818,16 +1689,96 @@ def reconciled_repository_receipt( version: str, workspace_root: Path, expected_receipt: dict[str, Any] | None, + repo_versions: dict[str, str] | None = None, ) -> dict[str, Any]: """Independently prove the repository effect before advancing reconciliation.""" + repo = str(plan_step["repo"]) + if spec.kind == "version": + if not isinstance(expected_receipt, dict): + raise ReleaseExecutionBlocked( + "Version reconciliation requires the preceding repository receipt." + ) + receipt = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_same_committed_repository_base(receipt, expected_receipt) + if receipt["worktree_clean"]: + raise ReleaseExecutionBlocked( + "The version metadata effect could not be verified." + ) + _require_release_owned_dirty_paths( + path=_repository_path(repo, workspace_root), + allow_core_bundle=False, + ) + if selected_repository_version_issues( + repo_versions={repo: version}, + workspace=workspace_root, + ): + raise ReleaseExecutionBlocked( + "The version metadata effect is not aligned." + ) + return receipt + if spec.kind == "core_bundle": + if not isinstance(expected_receipt, dict) or not repo_versions: + raise ReleaseExecutionBlocked( + "Core bundle reconciliation requires its frozen release inputs." + ) + receipt = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + _require_same_committed_repository_base(receipt, expected_receipt) + if receipt["worktree_clean"] or selected_release_webui_bundle_issues( + repo_versions=repo_versions, + workspace=workspace_root, + ): + raise ReleaseExecutionBlocked( + "The Core release-bundle effect could not be verified." + ) + _require_release_owned_dirty_paths( + path=_repository_path(repo, workspace_root), + allow_core_bundle=True, + ) + return receipt + if spec.kind == "commit": + if not isinstance(expected_receipt, dict): + raise ReleaseExecutionBlocked( + "Commit reconciliation requires the preceding repository receipt." + ) + receipt = repository_state_receipt( + repo=repo, + target_tag=f"v{version.removeprefix('v')}", + workspace_root=workspace_root, + ) + path = _repository_path(repo, workspace_root) + parent = git_text(path, "rev-parse", "--verify", "HEAD^") + message = git_text(path, "show", "-s", "--format=%s", "HEAD") + changed = _git_commit_changed_paths(path) + if ( + parent != expected_receipt.get("head") + or message != f"Release {repo} v{version.removeprefix('v')}" + or not receipt["worktree_clean"] + or any(not _release_metadata_path(item) for item in changed) + or selected_repository_version_issues( + repo_versions={repo: version}, + workspace=workspace_root, + ) + ): + raise ReleaseExecutionBlocked( + "The bounded release commit effect could not be verified." + ) + return receipt if spec.kind == "tag": if not isinstance(expected_receipt, dict): raise ReleaseExecutionBlocked( "Tag reconciliation requires the preceding repository receipt." ) receipt = repository_state_receipt( - repo=str(plan_step["repo"]), + repo=repo, target_tag=f"v{version.removeprefix('v')}", workspace_root=workspace_root, ) @@ -912,6 +1863,39 @@ def _require_same_repository_base( ) +def _require_same_committed_repository_base( + current: dict[str, Any], + expected: dict[str, Any], +) -> None: + ignored = {"worktree_clean", "worktree_sha256"} + if any( + current.get(key) != value + for key, value in expected.items() + if key not in ignored + ): + raise ReleaseExecutionBlocked( + "Repository committed identity changed during metadata mutation." + ) + + +def _git_commit_changed_paths(path: Path) -> tuple[str, ...]: + result = git( + path, + "diff-tree", + "--no-commit-id", + "--name-only", + "-r", + "-z", + "HEAD", + timeout=30, + ) + if result.returncode != 0: + raise ReleaseExecutionBlocked( + "Release commit paths cannot be inspected." + ) + return tuple(sorted({item for item in result.stdout.split("\0") if item})) + + def generate_catalog_candidate( *, candidate_root: Path, diff --git a/tools/release/govoplan_release/release_run.py b/tools/release/govoplan_release/release_run.py index f547062..4145274 100644 --- a/tools/release/govoplan_release/release_run.py +++ b/tools/release/govoplan_release/release_run.py @@ -2218,7 +2218,7 @@ def _validated_result_receipt( "Catalog publication receipt identity is invalid." ) return dict(value) - if set(value) != { + repository_fields = { "kind", "repo", "head", @@ -2228,7 +2228,8 @@ def _validated_result_receipt( "worktree_clean", "target_tag", "tag_object", - }: + } + if set(value) not in (repository_fields, repository_fields | {"worktree_sha256"}): raise ReleaseRunConflict("Repository result receipt is invalid.") if ( not isinstance(value.get("repo"), str) @@ -2241,6 +2242,10 @@ def _validated_result_receipt( or value.get("remote") != "origin" or not _is_sha256(value.get("remote_sha256")) or type(value.get("worktree_clean")) is not bool + or ( + "worktree_sha256" in value + and not _is_sha256(value.get("worktree_sha256")) + ) or not isinstance(value.get("target_tag"), str) or len(value["target_tag"]) > 160 or ( diff --git a/tools/release/govoplan_release/selective_planner.py b/tools/release/govoplan_release/selective_planner.py index abac48a..868ec78 100644 --- a/tools/release/govoplan_release/selective_planner.py +++ b/tools/release/govoplan_release/selective_planner.py @@ -25,6 +25,10 @@ from .version_alignment import ( selected_release_webui_bundle_issues, selected_repository_version_issues, ) +from .version_metadata import ( + VersionMetadataError, + version_metadata_mutations, +) def build_selective_release_plan( @@ -46,6 +50,7 @@ def build_selective_release_plan( ) for repo in repositories ) + units = dependency_ordered_units(units) units = apply_repository_version_gate( units, workspace=Path(dashboard.workspace_root) ) @@ -103,7 +108,35 @@ def apply_repository_version_gate( workspace: Path, ) -> tuple[ReleasePlanUnit, ...]: issues_by_repo: dict[str, list[ReleaseGateFinding]] = {} + version_update_supported_by_repo: dict[str, bool] = {} + deferred_core_lock_repos: set[str] = set() for unit in units: + version_update_supported = unit.current_version == unit.target_version + if unit.current_version and unit.current_version != unit.target_version: + try: + version_update_supported = bool( + version_metadata_mutations( + workspace / unit.repo, + target_version=unit.target_version, + ) + ) + except VersionMetadataError as exc: + issues_by_repo.setdefault(unit.repo, []).append( + ReleaseGateFinding( + code="repository_version_mutation_unsupported", + severity="blocker", + message=str(exc), + remediation=( + "Align the unsupported version declarations manually, " + "commit the reviewed result, then rebuild this plan." + ), + repo=unit.repo, + source="repository version metadata", + expected=unit.target_version, + actual=unit.current_version, + ) + ) + version_update_supported_by_repo[unit.repo] = version_update_supported try: issues = selected_repository_version_issues( repo_versions={unit.repo: unit.target_version}, @@ -130,6 +163,18 @@ def apply_repository_version_gate( ) continue for issue in issues: + if ( + version_update_supported + and issue.message + == "source version must match the requested release version" + ): + continue + if ( + unit.repo == "govoplan-core" + and issue.source.startswith("package-lock.release.json") + ): + deferred_core_lock_repos.add(unit.repo) + continue issues_by_repo.setdefault(issue.repo, []).append( ReleaseGateFinding( code="repository_version_alignment", @@ -161,17 +206,68 @@ def apply_repository_version_gate( source_preflight_ready=False, ) if unit.repo in issues_by_repo - else unit + else replace( + unit, + warnings=planned_version_warnings( + unit, + defer_core_lock=unit.repo in deferred_core_lock_repos, + ), + status=( + "attention" + if ( + ( + unit.current_version + and unit.current_version != unit.target_version + ) + or unit.repo in deferred_core_lock_repos + ) + and unit.status == "ready" + else unit.status + ), + source_preflight_ready=( + unit.source_preflight_ready + or ( + bool(unit.current_version) + and unit.current_version != unit.target_version + and version_update_supported_by_repo.get(unit.repo, False) + and not any( + finding.code == "worktree_dirty" + for finding in unit.gate_findings + ) + ) + ), + ) for unit in units ) +def planned_version_warnings( + unit: ReleasePlanUnit, + *, + defer_core_lock: bool, +) -> tuple[str, ...]: + warnings = list(unit.warnings) + if unit.current_version and unit.current_version != unit.target_version: + warnings.append( + "version metadata will be updated by the bounded release executor" + ) + if defer_core_lock: + warnings.append( + "Core release lock metadata will be regenerated by the bounded " + "bundle executor" + ) + return tuple(warnings) + + def apply_release_webui_bundle_gate( units: tuple[ReleasePlanUnit, ...], *, workspace: Path, ) -> tuple[ReleasePlanUnit, ...]: issues_by_repo: dict[str, list[ReleaseGateFinding]] = {} + selected = {unit.repo for unit in units} + core_selected = "govoplan-core" in selected + planned_bundle_repos: set[str] = set() try: issues = selected_release_webui_bundle_issues( repo_versions={unit.repo: unit.target_version for unit in units}, @@ -200,6 +296,9 @@ def apply_release_webui_bundle_gate( ) issues = () for issue in issues: + if core_selected and issue.repo in selected and issue.repo != "govoplan-core": + planned_bundle_repos.add(issue.repo) + continue issues_by_repo.setdefault(issue.repo, []).append( ReleaseGateFinding( code="release_webui_composition_alignment", @@ -230,7 +329,25 @@ def apply_release_webui_bundle_gate( source_preflight_ready=False, ) if unit.repo in issues_by_repo - else unit + else replace( + unit, + warnings=unit.warnings + + ( + ( + "Core release WebUI references will be regenerated after " + "the selected module tags are created" + ), + ) + if unit.repo == "govoplan-core" and planned_bundle_repos + else unit.warnings, + status=( + "attention" + if unit.repo == "govoplan-core" + and planned_bundle_repos + and unit.status == "ready" + else unit.status + ), + ) for unit in units ) @@ -408,6 +525,7 @@ def build_unit( status="blocked" if blockers else "attention" if warnings else "ready", blockers=tuple(blockers), warnings=tuple(warnings), + capabilities=repository_capabilities(repo, contracts=contracts), provides_interfaces=provides, requires_interfaces=requires, gate_findings=tuple(findings), @@ -416,11 +534,83 @@ def build_unit( and not repo.dirty and repo.has_head and bool(repo.branch) - and current_version == resolved_target + and current_version is not None ), ) +def repository_capabilities( + repo: RepositorySnapshot, + *, + contracts: ModuleContractSnapshot | None, +) -> tuple[str, ...]: + """Describe the release operations applicable to one repository shape.""" + + root = Path(repo.absolute_path) + capabilities = {"git-source"} + if (root / "pyproject.toml").is_file(): + capabilities.add("python-package") + if (root / "package.json").is_file() or (root / "webui/package.json").is_file(): + capabilities.add("webui-package") + if contracts is not None: + capabilities.add("module-manifest") + if ( + (root / "alembic/versions").is_dir() + or (root / "src").is_dir() + and any((root / "src").glob("**/migrations/versions")) + ): + capabilities.add("database-migrations") + if (root / "docs").is_dir() or repo.spec.subtype in {"docs", "documentation"}: + capabilities.add("documentation") + if repo.spec.name == "govoplan-core": + capabilities.add("core-release-bundle") + if repo.spec.name == "govoplan": + capabilities.add("release-control-plane") + return tuple(sorted(capabilities)) + + +def dependency_ordered_units( + units: tuple[ReleasePlanUnit, ...], +) -> tuple[ReleasePlanUnit, ...]: + """Order module providers before consumers while keeping Core last.""" + + by_repo = {unit.repo: unit for unit in units} + providers: dict[str, set[str]] = {} + for unit in units: + if unit.repo == "govoplan-core": + continue + for provided in unit.provides_interfaces: + providers.setdefault(provided.name, set()).add(unit.repo) + + dependencies: dict[str, set[str]] = {unit.repo: set() for unit in units} + for unit in units: + for requirement in unit.requires_interfaces: + dependencies[unit.repo].update( + provider + for provider in providers.get(requirement.name, set()) + if provider != unit.repo + ) + if "govoplan-core" in dependencies: + dependencies["govoplan-core"].update( + repo for repo in by_repo if repo != "govoplan-core" + ) + + ordered: list[ReleasePlanUnit] = [] + remaining = set(by_repo) + while remaining: + ready = sorted( + repo + for repo in remaining + if not (dependencies[repo] & remaining) + ) + if not ready: + ready = [sorted(remaining)[0]] + for repo in ready: + ordered.append(by_repo[repo]) + remaining.remove(repo) + return tuple(ordered) + + def gate_finding( *, repo: str, @@ -501,24 +691,38 @@ def dry_run_steps( ) -> tuple[ReleasePlanStep, ...]: steps: list[ReleasePlanStep] = [] snapshots = {repo.spec.name: repo for repo in dashboard.repositories} - selected_names = {unit.repo for unit in units} - if "govoplan-core" in selected_names and len(selected_names) > 1: - steps.append( - ReleasePlanStep( - id="release:cross-repository-ordering", - title="Resolve module/Core release ordering", - detail=( - "A combined Core/module release needs a dependency-aware DAG: " - "local module tags, Core release-lock regeneration and commit, " - "Core tag, full alignment, then atomic pushes. The current linear " - "executor must not publish a module before that lock is committed." - ), - command=None, - cwd=dashboard.meta_root, - mutating=True, - status="needs-executor", + core_unit = next((unit for unit in units if unit.repo == "govoplan-core"), None) + non_core_units = tuple(unit for unit in units if unit.repo != "govoplan-core") + bundle_repos: tuple[str, ...] = () + core_lock_regeneration = False + if core_unit is not None: + try: + bundle_repos = tuple( + sorted( + { + issue.repo + for issue in selected_release_webui_bundle_issues( + repo_versions={ + unit.repo: unit.target_version for unit in units + }, + workspace=Path(dashboard.workspace_root), + ) + if issue.repo != "govoplan-core" + } + ) ) - ) + core_lock_regeneration = bool(bundle_repos) or any( + issue.source.startswith("package-lock.release.json") + for issue in selected_repository_version_issues( + repo_versions={ + "govoplan-core": core_unit.target_version, + }, + workspace=Path(dashboard.workspace_root), + ) + ) + except Exception: # noqa: BLE001 - a structured gate already reports it. + bundle_repos = () + for unit in units: steps.append( ReleasePlanStep( @@ -531,32 +735,74 @@ def dry_run_steps( repo=unit.repo, ) ) + + for unit in units: if unit.current_version != unit.target_version: + try: + paths = tuple( + mutation.path + for mutation in version_metadata_mutations( + Path(dashboard.workspace_root) / unit.repo, + target_version=unit.target_version, + ) + ) + except VersionMetadataError: + paths = () + snapshot = snapshots.get(unit.repo) + version_supported = bool( + paths and snapshot is not None and not snapshot.dirty + ) steps.append( ReleasePlanStep( id=f"{unit.repo}:version", title=f"Update {unit.repo} version metadata", - detail=f"Prepare version files and manifest metadata for {unit.target_version}.", - command=None, + detail=( + f"Update only the recognized version declarations for " + f"{unit.target_version}: {', '.join(paths) or 'unsupported metadata'}." + ), + command=( + "bounded version metadata update " + + " ".join(shlex.quote(path) for path in paths) + ) + if paths + else None, cwd=f"{dashboard.workspace_root}/{unit.repo}", mutating=True, repo=unit.repo, - status="needs-executor", + status="planned" if version_supported else "needs-executor", ) ) + + for unit in non_core_units: snapshot = snapshots.get(unit.repo) - if unit.current_version != unit.target_version or ( + needs_commit = unit.current_version != unit.target_version or ( snapshot is not None and snapshot.dirty - ): + ) + if needs_commit: + commit_supported = bool( + unit.current_version != unit.target_version + and snapshot is not None + and not snapshot.dirty + ) steps.append( ReleasePlanStep( id=f"{unit.repo}:commit", title=f"Commit {unit.repo} release changes", - detail="Commit reviewed changes for this release unit only.", - command=f"git add -A && git commit -m {shlex.quote(f'Release {unit.repo} {unit.target_tag}')}", + detail=( + "Commit only the receipt-bound files written by the version " + "executor." + if commit_supported + else "Pre-existing worktree changes require review outside the release executor." + ), + command=( + f"git commit -m {shlex.quote(f'Release {unit.repo} {unit.target_tag}')}" + if commit_supported + else None + ), cwd=f"{dashboard.workspace_root}/{unit.repo}", mutating=True, repo=unit.repo, + status="planned" if commit_supported else "needs-executor", ) ) steps.append( @@ -570,6 +816,97 @@ def dry_run_steps( repo=unit.repo, ) ) + + if core_unit is not None: + if core_lock_regeneration: + steps.append( + ReleasePlanStep( + id="govoplan-core:bundle", + title="Regenerate the Core release WebUI bundle", + detail=( + "After local module tags exist, update and lock the exact " + "selected WebUI references" + + ( + " for: " + ", ".join(bundle_repos) + if bundle_repos + else "" + ) + + ", and reconcile package/lock metadata." + ), + command=( + "tools/release/generate-release-lock.sh " + + " ".join( + f"--local-git-repo {shlex.quote(str(Path(dashboard.workspace_root) / repo))}" + for repo in bundle_repos + ) + ), + cwd=dashboard.meta_root, + mutating=True, + repo="govoplan-core", + ) + ) + core_snapshot = snapshots.get(core_unit.repo) + core_needs_commit = ( + core_unit.current_version != core_unit.target_version + or core_lock_regeneration + or (core_snapshot is not None and core_snapshot.dirty) + ) + if core_needs_commit: + core_commit_supported = bool( + core_snapshot is not None + and not core_snapshot.dirty + and ( + core_unit.current_version != core_unit.target_version + or core_lock_regeneration + ) + ) + steps.append( + ReleasePlanStep( + id="govoplan-core:commit", + title="Commit govoplan-core release changes", + detail=( + "Commit only the receipt-bound Core version and release-bundle files." + if core_commit_supported + else "Pre-existing Core worktree changes require review outside the release executor." + ), + command=( + f"git commit -m {shlex.quote(f'Release govoplan-core {core_unit.target_tag}')}" + if core_commit_supported + else None + ), + cwd=f"{dashboard.workspace_root}/govoplan-core", + mutating=True, + repo="govoplan-core", + status="planned" if core_commit_supported else "needs-executor", + ) + ) + steps.append( + ReleasePlanStep( + id="govoplan-core:tag", + title=f"Create govoplan-core tag {core_unit.target_tag}", + detail="Create the Core tag only after selected module tags and the release lock are verified.", + command=f"git tag -a {shlex.quote(core_unit.target_tag)} -m {shlex.quote(f'Release govoplan-core {core_unit.target_tag}')}", + cwd=f"{dashboard.workspace_root}/govoplan-core", + mutating=True, + repo="govoplan-core", + ) + ) + + if units: + steps.append( + ReleasePlanStep( + id="release:alignment", + title="Verify frozen source and release composition", + detail=( + "Re-run version, contract, tag, and Core WebUI composition gates " + "after local mutations and before any remote publication." + ), + command="receipt-bound release alignment", + cwd=dashboard.meta_root, + ) + ) + + for unit in units: steps.append( ReleasePlanStep( id=f"{unit.repo}:push", @@ -640,6 +977,20 @@ def dry_run_steps( status="planned", ) ) + steps.append( + ReleasePlanStep( + id="release:install-verify", + title="Verify candidate package installation", + detail=( + "Install every selected candidate wheel into a private " + "temporary target without network access or dependencies, " + "then verify installed package metadata against the frozen plan." + ), + command="python -m pip install --no-index --no-deps ", + cwd=dashboard.meta_root, + status="planned", + ) + ) return tuple(steps) diff --git a/tools/release/govoplan_release/version_metadata.py b/tools/release/govoplan_release/version_metadata.py new file mode 100644 index 0000000..67742f0 --- /dev/null +++ b/tools/release/govoplan_release/version_metadata.py @@ -0,0 +1,387 @@ +"""Deterministic, bounded updates for repository-owned version metadata.""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import tempfile +import tomllib + + +MAX_VERSION_FILES = 64 +MAX_VERSION_FILE_BYTES = 2 * 1024 * 1024 + + +class VersionMetadataError(RuntimeError): + """Raised when release version metadata cannot be changed safely.""" + + +@dataclass(frozen=True, slots=True) +class VersionFileMutation: + path: str + before: bytes + after: bytes + + +def version_metadata_mutations( + repo_path: Path, + *, + target_version: str, +) -> tuple[VersionFileMutation, ...]: + """Render all recognized repository version files without writing them.""" + + version = target_version.removeprefix("v") + candidates: list[tuple[Path, str]] = [] + if (repo_path / "pyproject.toml").is_file(): + candidates.append((repo_path / "pyproject.toml", "pyproject")) + for package_name, lock_name in ( + ("package.json", "package-lock.json"), + ("webui/package.json", "webui/package-lock.json"), + ("webui/package.release.json", "webui/package-lock.release.json"), + ): + package = repo_path / package_name + if not package.is_file(): + continue + candidates.append((package, "package")) + lock = repo_path / lock_name + if lock.is_file(): + candidates.append((lock, "lock")) + src = repo_path / "src" + if src.is_dir(): + candidates.extend( + (path, "manifest") + for path in sorted(src.glob("**/backend/manifest.py")) + if path.is_file() + ) + candidates.extend( + (path, "package_init") + for path in sorted(src.glob("*/__init__.py")) + if path.is_file() + ) + if len(candidates) > MAX_VERSION_FILES: + raise VersionMetadataError("Repository version metadata exceeds its file bound.") + + mutations: list[VersionFileMutation] = [] + declarations = 0 + for path, kind in candidates: + before = _read_bounded(path) + if kind == "pyproject": + after, found = _render_pyproject(before, version=version) + elif kind == "package": + after, found = _render_package_json(before, version=version) + elif kind == "lock": + after, found = _render_package_lock(before, version=version) + elif kind == "manifest": + after, found = _render_python_version( + before, + path=path, + target_name="ModuleManifest", + keyword_name="version", + version=version, + ) + else: + after, found = _render_python_assignment( + before, + path=path, + assignment_name="__version__", + version=version, + ) + if not found: + continue + declarations += 1 + if before != after: + mutations.append( + VersionFileMutation( + path=path.relative_to(repo_path).as_posix(), + before=before, + after=after, + ) + ) + if declarations == 0: + raise VersionMetadataError( + "Repository has no supported version declaration to update." + ) + return tuple(mutations) + + +def apply_version_metadata_mutations( + repo_path: Path, + *, + target_version: str, +) -> tuple[str, ...]: + """Apply one deterministic version update, rolling back on write failure.""" + + mutations = version_metadata_mutations( + repo_path, + target_version=target_version, + ) + written: list[VersionFileMutation] = [] + try: + for mutation in mutations: + destination = repo_path / mutation.path + if _read_bounded(destination) != mutation.before: + raise VersionMetadataError( + f"Version metadata changed before update: {mutation.path}" + ) + _atomic_write(destination, mutation.after) + written.append(mutation) + except Exception: + for mutation in reversed(written): + _atomic_write(repo_path / mutation.path, mutation.before) + raise + return tuple(mutation.path for mutation in mutations) + + +def version_metadata_paths( + repo_path: Path, + *, + target_version: str, +) -> tuple[str, ...]: + """Return the exact paths a deterministic target-version update owns.""" + + return tuple( + mutation.path + for mutation in version_metadata_mutations( + repo_path, + target_version=target_version, + ) + ) + + +def _read_bounded(path: Path) -> bytes: + try: + metadata = path.lstat() + except OSError as exc: + raise VersionMetadataError( + f"Version metadata is unavailable: {path.name}" + ) from exc + if not path.is_file() or path.is_symlink(): + raise VersionMetadataError( + f"Version metadata must be a regular file: {path.name}" + ) + if metadata.st_size > MAX_VERSION_FILE_BYTES: + raise VersionMetadataError( + f"Version metadata exceeds its size bound: {path.name}" + ) + try: + return path.read_bytes() + except OSError as exc: + raise VersionMetadataError( + f"Version metadata could not be read: {path.name}" + ) from exc + + +def _decode(payload: bytes, *, path: Path | None = None) -> str: + try: + return payload.decode("utf-8") + except UnicodeDecodeError as exc: + label = path.name if path is not None else "file" + raise VersionMetadataError( + f"Version metadata is not UTF-8: {label}" + ) from exc + + +def _render_pyproject(payload: bytes, *, version: str) -> tuple[bytes, bool]: + text = _decode(payload) + try: + parsed = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + raise VersionMetadataError("pyproject.toml is malformed.") from exc + project = parsed.get("project") + if not isinstance(project, dict) or not isinstance(project.get("version"), str): + return payload, False + section = re.search( + r"(?ms)^\[project\][^\n]*\n(?P.*?)(?=^\[[^\n]+\]|\Z)", + text, + ) + if section is None: + raise VersionMetadataError("pyproject.toml has no editable [project] block.") + body = section.group("body") + match = re.search( + r'(?m)^(?P\s*version\s*=\s*)(?P["\'])(?P[^"\']+)(?P=quote)(?P\s*(?:#.*)?)$', + body, + ) + if match is None: + raise VersionMetadataError( + "pyproject.toml project.version is not a supported literal." + ) + start = section.start("body") + match.start("value") + end = section.start("body") + match.end("value") + rendered = f"{text[:start]}{version}{text[end:]}" + tomllib.loads(rendered) + return rendered.encode("utf-8"), True + + +def _render_package_json(payload: bytes, *, version: str) -> tuple[bytes, bool]: + data = _json_object(payload, label="package metadata") + if not isinstance(data.get("version"), str): + return payload, False + data["version"] = version + return _json_bytes(data), True + + +def _render_package_lock(payload: bytes, *, version: str) -> tuple[bytes, bool]: + data = _json_object(payload, label="package lock") + found = False + if isinstance(data.get("version"), str): + data["version"] = version + found = True + packages = data.get("packages") + root = packages.get("") if isinstance(packages, dict) else None + if isinstance(root, dict) and isinstance(root.get("version"), str): + root["version"] = version + found = True + return (_json_bytes(data), True) if found else (payload, False) + + +def _json_object(payload: bytes, *, label: str) -> dict[str, object]: + try: + value = json.loads(_decode(payload)) + except json.JSONDecodeError as exc: + raise VersionMetadataError(f"{label.capitalize()} is malformed.") from exc + if not isinstance(value, dict): + raise VersionMetadataError(f"{label.capitalize()} must be an object.") + return value + + +def _json_bytes(value: dict[str, object]) -> bytes: + return ( + json.dumps(value, indent=2, ensure_ascii=True, separators=(",", ": ")) + + "\n" + ).encode("utf-8") + + +def _render_python_version( + payload: bytes, + *, + path: Path, + target_name: str, + keyword_name: str, + version: str, +) -> tuple[bytes, bool]: + text = _decode(payload, path=path) + try: + tree = ast.parse(text, filename=str(path)) + except SyntaxError as exc: + raise VersionMetadataError(f"Python metadata is malformed: {path.name}") from exc + values: list[ast.Constant] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or _call_name(node.func) != target_name: + continue + for keyword in node.keywords: + if ( + keyword.arg == keyword_name + and isinstance(keyword.value, ast.Constant) + and isinstance(keyword.value.value, str) + ): + values.append(keyword.value) + if not values: + return payload, False + if len(values) != 1: + raise VersionMetadataError( + f"Python metadata has multiple {target_name}.{keyword_name} values: {path.name}" + ) + return _replace_python_literal(text, values[0], version=version, path=path), True + + +def _render_python_assignment( + payload: bytes, + *, + path: Path, + assignment_name: str, + version: str, +) -> tuple[bytes, bool]: + text = _decode(payload, path=path) + try: + tree = ast.parse(text, filename=str(path)) + except SyntaxError as exc: + raise VersionMetadataError(f"Python metadata is malformed: {path.name}") from exc + values: list[ast.Constant] = [] + for node in tree.body: + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + continue + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + value = node.value + if ( + any( + isinstance(target, ast.Name) and target.id == assignment_name + for target in targets + ) + and isinstance(value, ast.Constant) + and isinstance(value.value, str) + ): + values.append(value) + if not values: + return payload, False + if len(values) != 1: + raise VersionMetadataError( + f"Python metadata has multiple {assignment_name} assignments: {path.name}" + ) + return _replace_python_literal(text, values[0], version=version, path=path), True + + +def _replace_python_literal( + text: str, + node: ast.Constant, + *, + version: str, + path: Path, +) -> bytes: + if ( + node.lineno != node.end_lineno + or node.end_col_offset is None + or node.col_offset < 0 + ): + raise VersionMetadataError( + f"Python version declaration must be a single-line literal: {path.name}" + ) + lines = text.splitlines(keepends=True) + line = lines[node.lineno - 1] + lines[node.lineno - 1] = ( + line[: node.col_offset] + + json.dumps(version) + + line[node.end_col_offset :] + ) + rendered = "".join(lines) + ast.parse(rendered, filename=str(path)) + return rendered.encode("utf-8") + + +def _call_name(node: ast.expr) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + +def _atomic_write(path: Path, payload: bytes) -> None: + metadata = path.stat() + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, metadata.st_mode & 0o777) + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + temporary.replace(path) + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + except Exception: + try: + os.close(descriptor) + except OSError: + pass + temporary.unlink(missing_ok=True) + raise diff --git a/tools/release/release-migration-audit.py b/tools/release/release-migration-audit.py index e4b0df4..00e4536 100644 --- a/tools/release/release-migration-audit.py +++ b/tools/release/release-migration-audit.py @@ -170,18 +170,30 @@ def owner_for_versions_dir(versions_dir: Path) -> str: def parse_migration_file(owner: str, path: Path) -> Migration | None: tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) values: dict[str, Any] = {} + wrapped: Migration | None = None + release_peer = path.parent.parent / "versions" / path.name + if path.parent.name == "dev_versions" and release_peer.is_file(): + wrapped = parse_migration_file(owner, release_peer) for statement in tree.body: if isinstance(statement, ast.Assign): for target in statement.targets: if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "depends_on", "branch_labels"}: - values[target.id] = ast.literal_eval(statement.value) + values[target.id] = _migration_assignment_value( + target.id, + statement.value, + wrapped=wrapped, + ) elif ( isinstance(statement, ast.AnnAssign) and isinstance(statement.target, ast.Name) and statement.target.id in {"revision", "down_revision", "depends_on", "branch_labels"} and statement.value is not None ): - values[statement.target.id] = ast.literal_eval(statement.value) + values[statement.target.id] = _migration_assignment_value( + statement.target.id, + statement.value, + wrapped=wrapped, + ) revision = values.get("revision") if not isinstance(revision, str): return None @@ -195,6 +207,31 @@ def parse_migration_file(owner: str, path: Path) -> Migration | None: ) +def _migration_assignment_value( + name: str, + value: ast.expr, + *, + wrapped: Migration | None, +) -> Any: + try: + return ast.literal_eval(value) + except (ValueError, TypeError): + if ( + wrapped is None + or not isinstance(value, ast.Attribute) + or not isinstance(value.value, ast.Name) + or value.value.id != "_migration" + or value.attr != name + ): + return None + return { + "revision": wrapped.revision, + "down_revision": wrapped.down_revisions, + "depends_on": wrapped.depends_on, + "branch_labels": wrapped.branch_labels, + }[name] + + def _normalize_revision_tuple(value: Any) -> tuple[str, ...]: if value is None: return () diff --git a/tools/release/server/app.py b/tools/release/server/app.py index 30c128f..467a4b9 100644 --- a/tools/release/server/app.py +++ b/tools/release/server/app.py @@ -45,7 +45,9 @@ from govoplan_release.release_execution import ( verify_catalog_publication_precondition, verify_repository_preflight_binding, verify_repository_step_precondition, + verify_release_alignment, verify_release_runtime_binding, + verify_candidate_installation, ) from govoplan_release.release_run import ( ReleaseRunConflict, @@ -532,7 +534,13 @@ def create_app( ) == "interrupted": plan_step = release_run_plan_step(run, step_id) spec = executor_spec(plan_step) - if spec is not None and spec.kind in {"tag", "push"}: + if spec is not None and spec.kind in { + "version", + "commit", + "core_bundle", + "tag", + "push", + }: repo = str(plan_step["repo"]) receipt = reconciled_repository_receipt( spec=spec, @@ -542,6 +550,7 @@ def create_app( expected_receipt=preceding_repository_receipt( run, step_id=step_id, repo=repo ), + repo_versions=run["immutable"]["input"]["repo_versions"], ) return release_run_view( app.state.release_runs.reconcile_step( @@ -637,7 +646,15 @@ def create_app( plan_step=plan_step, workspace_root=app.state.workspace_root, ) - if spec.kind in {"preflight", "tag", "push"}: + repository_kinds = { + "preflight", + "version", + "commit", + "core_bundle", + "tag", + "push", + } + if spec.kind in repository_kinds: repo = str(plan_step["repo"]) version = run["immutable"]["input"]["repo_versions"][repo] expected_repository_receipt = preceding_repository_receipt( @@ -659,7 +676,21 @@ def create_app( ) receipt = None - if spec.kind in {"preflight", "tag", "push"}: + source_receipts = { + repo: source_receipt + for repo in run["immutable"]["input"]["repo_versions"] + if isinstance( + ( + source_receipt := preceding_repository_receipt( + run, + step_id=step_id, + repo=repo, + ) + ), + dict, + ) + } + if spec.kind in repository_kinds: result, receipt = execute_repository_step( spec=spec, plan_step=plan_step, @@ -667,6 +698,13 @@ def create_app( workspace_root=app.state.workspace_root, remote=DURABLE_REMOTE, expected_receipt=expected_repository_receipt, + source_receipts=source_receipts, + ) + elif spec.kind == "alignment": + result = verify_release_alignment( + repo_versions=run["immutable"]["input"]["repo_versions"], + workspace_root=app.state.workspace_root, + source_receipts=source_receipts, ) elif spec.kind == "catalog_generate": attempt_fingerprint = ( @@ -690,18 +728,7 @@ def create_app( signing_keys=signing_keys, remote=DURABLE_REMOTE, check_public=run["immutable"]["input"]["public_catalog"], - source_receipts={ - repo: receipt - for repo in run["immutable"]["input"]["repo_versions"] - if isinstance( - ( - receipt := preceding_repository_receipt( - run, step_id=step_id, repo=repo - ) - ), - dict, - ) - }, + source_receipts=source_receipts, base_catalog=base_catalog, base_keyring=base_keyring, ) @@ -738,6 +765,15 @@ def create_app( "Published candidate changed while its effect was in flight; " "reconcile the immutable website commit and remote tag." ) from exc + elif spec.kind == "install_verify": + candidate_path = verified_run_candidate( + run, + candidate_root=app.state.release_candidate_root, + ) + result = verify_candidate_installation( + candidate_path=candidate_path, + repo_versions=run["immutable"]["input"]["repo_versions"], + ) else: raise ReleaseRunConflict("Release executor mapping is incomplete.") diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html index 5594fd2..e7d7838 100644 --- a/tools/release/webui/index.html +++ b/tools/release/webui/index.html @@ -943,10 +943,10 @@

5. Installation Verification

- external gate + receipt-bound local gate
-

The signed catalog and Python artifacts are produced by the durable candidate step. End-to-end installation verification still runs in release integration CI and is not yet a bounded local-console executor.

+

The durable verification step installs every selected candidate wheel into a private temporary target with network and dependency resolution disabled, then compares installed package metadata with the frozen release plan. Deployment-specific integration checks remain part of CI.

@@ -1783,7 +1783,7 @@ target: "releaseRunSection", run, plan, - match: (step) => /:(preflight|tag|push)$/.test(step.id || ""), + match: (step) => /:(preflight|version|commit|bundle|tag|push)$/.test(step.id || "") || step.id === "release:alignment", waitingStatus: "create run", }), runWorkflowPhase({ @@ -1804,13 +1804,15 @@ match: (step) => step.id === "catalog:validate-sign-publish", waitingStatus: "waiting", }), - { + runWorkflowPhase({ id: "verify", label: "Verify", target: "installVerificationSection", - state: run?.state?.status === "completed" ? "unavailable" : "locked", - status: run?.state?.status === "completed" ? "CI gate" : "waiting", - }, + run, + plan, + match: (step) => step.id === "release:install-verify", + waitingStatus: "waiting", + }), ]; const firstOpen = phases.findIndex((phase) => ["current", "blocked", "unavailable"].includes(phase.state)); if (firstOpen >= 0) { @@ -1924,10 +1926,10 @@ }; } else if (run.state?.status === "completed") { guidance = { - title: "Run installation verification", - detail: "Every durable source, package, signing, and publication step has a recorded success receipt.", - remediation: "End-to-end installation verification is still an external release-integration CI gate.", - button: "Review verification gate", + title: "Release run completed", + detail: "Every durable source, package, signing, publication, and local installation-verification step has a recorded success.", + remediation: "Review the run receipts and continue with deployment-specific integration checks where required.", + button: "Review completed run", action: { type: "scroll", target: "installVerificationSection" }, }; } else { @@ -2389,9 +2391,12 @@ const blockers = unit.blockers.length ? `

${escapeHtml(unit.blockers.join("; "))}

` : ""; const warnings = unit.warnings.length ? `

${escapeHtml(unit.warnings.join("; "))}

` : ""; const kind = unit.status === "blocked" ? "block" : unit.status === "attention" ? "warn" : "ok"; + const capabilities = Array.isArray(unit.capabilities) && unit.capabilities.length + ? `

Capabilities: ${escapeHtml(unit.capabilities.join(", "))}

` + : ""; return `

${pill(unit.status, kind)} ${escapeHtml(unit.repo)} ${escapeHtml(unit.current_version || "-")} -> ${escapeHtml(unit.target_version)}

- ${blockers}${warnings} + ${capabilities}${blockers}${warnings}
`; }).join(""); const compatibilityHtml = compatibility.map((issue) => {