From a15a74c54cf73811d26d5911aa4b540a6e76c1c2 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Tue, 21 Jul 2026 12:25:06 +0200 Subject: [PATCH] feat(release): enforce aligned trusted publications --- docs/RELEASE_CONSOLE.md | 18 + tests/test_release_catalog_publication.py | 99 +++ tests/test_release_doctor.py | 16 + tests/test_release_entrypoint_gates.py | 60 ++ tests/test_version_alignment.py | 184 ++++++ tools/checks/check-release-integration.sh | 7 + tools/checks/check-version-alignment.py | 130 ++++ tools/release/generate-release-catalog.py | 15 + tools/release/generate-release-lock.sh | 32 +- tools/release/govoplan_release/git_state.py | 16 + tools/release/govoplan_release/model.py | 9 +- tools/release/govoplan_release/publisher.py | 55 +- .../govoplan_release/selective_catalog.py | 30 +- .../govoplan_release/selective_planner.py | 13 + .../govoplan_release/version_alignment.py | 572 ++++++++++++++++++ tools/release/publish-release-catalog.sh | 1 + tools/release/push-release-tag.sh | 151 ++++- tools/release/release-doctor.py | 2 +- 18 files changed, 1359 insertions(+), 51 deletions(-) create mode 100644 tests/test_release_catalog_publication.py create mode 100644 tests/test_release_entrypoint_gates.py create mode 100644 tests/test_version_alignment.py create mode 100644 tools/checks/check-version-alignment.py create mode 100644 tools/release/govoplan_release/version_alignment.py diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md index 2c11d36..c7c311a 100644 --- a/docs/RELEASE_CONSOLE.md +++ b/docs/RELEASE_CONSOLE.md @@ -151,6 +151,24 @@ forcing every repository to the same tag, but channel publication must preserve the unchanged package versions, validate interface compatibility, sign the updated catalog, and keep the published keyring healthy. +Release integration also enforces repository and composition version alignment +and generates a CycloneDX SBOM from the resolved Python environment and the +release WebUI lockfile. Catalog publication should attach that immutable SBOM +and its digest to the corresponding Core/composition release. + +Addresses and Notifications have current WebUI source contributions but are +not part of the pinned v0.1.8 WebUI release composition because their v0.1.8 +tags predate those packages. They re-enter the release composition only through +new, immutable module tags whose backend, manifest, frontend, and lock metadata +pass the alignment gate. + +Candidate publication verifies signatures against the already published +keyring, not against keys supplied only by the candidate. A changed keyring +must have its canonical SHA-256 embedded in the signed catalog. The public +module-directory files are regenerated from that verified catalog and keyring +at publication time; candidate-supplied directory files are never copied as +authoritative provenance. + ## Published Module Directory The target release repository is an online, browsable module directory. The diff --git a/tests/test_release_catalog_publication.py b/tests/test_release_catalog_publication.py new file mode 100644 index 0000000..70ac419 --- /dev/null +++ b/tests/test_release_catalog_publication.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + + +META_ROOT = Path(__file__).resolve().parents[1] +RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release" +if str(RELEASE_TOOLS_ROOT) not in sys.path: + sys.path.insert(0, str(RELEASE_TOOLS_ROOT)) + +from govoplan_release.publisher import publish_catalog_candidate # noqa: E402 + + +class ReleaseCatalogPublicationTests(unittest.TestCase): + def test_publication_requires_an_existing_trust_anchor(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = self._candidate(root, key="candidate-key") + web_root = root / "website" + web_root.mkdir() + + with patch( + "govoplan_release.publisher.validate_module_package_catalog", + return_value={"valid": True, "warnings": [], "error": None}, + ): + result = publish_catalog_candidate( + candidate_dir=candidate, + web_root=web_root, + workspace_root=root, + ) + + self.assertEqual("blocked", result.status) + self.assertIn("publication trust anchor is missing", " ".join(result.notes)) + + def test_publication_rejects_rebinding_an_existing_key_id(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = self._candidate(root, key="replacement-key") + web_root = root / "website" + target_keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json" + target_keyring.parent.mkdir(parents=True) + target_keyring.write_text(json.dumps(self._keyring("trusted-key"))) + + with patch( + "govoplan_release.publisher.validate_module_package_catalog", + return_value={"valid": True, "warnings": [], "error": None}, + ): + result = publish_catalog_candidate( + candidate_dir=candidate, + web_root=web_root, + workspace_root=root, + ) + + self.assertEqual("blocked", result.status) + self.assertIn("changes the public key", " ".join(result.notes)) + + @staticmethod + def _candidate(root: Path, *, key: str) -> Path: + candidate = root / "candidate" + channel = candidate / "channels" + channel.mkdir(parents=True) + channel.joinpath("stable.json").write_text( + json.dumps( + { + "channel": "stable", + "core_release": { + "version": "1.2.3", + "python_ref": ( + "govoplan-core @ git+ssh://git@example.test/acme/" + "govoplan-core.git@v1.2.3" + ), + }, + "modules": [], + } + ) + ) + candidate.joinpath("keyring.json").write_text(json.dumps(ReleaseCatalogPublicationTests._keyring(key))) + return candidate + + @staticmethod + def _keyring(key: str) -> dict[str, object]: + return { + "keys": [ + { + "key_id": "release-key", + "status": "active", + "public_key": key, + } + ] + } + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_doctor.py b/tests/test_release_doctor.py index b5f5ff6..1038bd1 100644 --- a/tests/test_release_doctor.py +++ b/tests/test_release_doctor.py @@ -51,6 +51,22 @@ class ReleaseDoctorTests(unittest.TestCase): self.assertEqual("repo-dirty", findings[0].check_id) self.assertIn("git status --short", [item.command for item in findings[0].commands]) + def test_repo_version_mismatch_blocks_release(self) -> None: + doctor = load_doctor_module() + repo = doctor.RepoState( + name="govoplan-files", + path="/tmp/govoplan-files", + exists=True, + pyproject_version="0.1.9", + package_version="0.1.8", + webui_package_version="0.1.9", + ) + + findings = doctor.repo_findings((repo,), target_version="0.1.9", target_tag="v0.1.9") + + mismatch = next(item for item in findings if item.check_id == "repo-version-mismatch") + self.assertEqual("blocker", mismatch.severity) + def test_dubious_ownership_errors_suggest_safe_directory_command(self) -> None: doctor = load_doctor_module() repo = doctor.RepoState( diff --git a/tests/test_release_entrypoint_gates.py b/tests/test_release_entrypoint_gates.py new file mode 100644 index 0000000..d413a12 --- /dev/null +++ b/tests/test_release_entrypoint_gates.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import json +from pathlib import Path +import unittest + + +META_ROOT = Path(__file__).resolve().parents[1] + + +class ReleaseEntrypointGateTests(unittest.TestCase): + def test_lockstep_release_includes_every_registered_module_and_connector(self) -> None: + script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text() + registry = json.loads((META_ROOT / "repositories.json").read_text()) + + expected = { + item["name"] + for item in registry["repositories"] + if item["category"] not in {"system", "website"} + } + missing = sorted(repo for repo in expected if f'$PARENT/{repo}"' not in script) + + self.assertEqual([], missing) + + def test_lockstep_release_runs_source_and_full_gates_before_remote_push(self) -> None: + script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text() + workflow = script.rsplit("\nconfirm_release\n", 1)[1] + + source_gate = workflow.index("run_version_alignment_gate source") + first_commit = workflow.index('run git -C "$repo" commit') + lock_generation = workflow.index("generate_release_lock") + full_gate = workflow.index("run_version_alignment_gate", source_gate + 1) + first_push = workflow.index('run git -C "$repo" push') + + self.assertLess(source_gate, first_commit) + self.assertLess(first_commit, lock_generation) + self.assertLess(lock_generation, full_gate) + self.assertLess(full_gate, first_push) + + def test_source_catalog_generator_enforces_explicit_repo_versions(self) -> None: + script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text() + + gate = script.index("selected_repository_version_issues(") + write = script.index("output.write_text(") + + self.assertLess(gate, write) + + def test_candidate_publication_uses_existing_keyring_as_trust_anchor(self) -> None: + publisher = (META_ROOT / "tools" / "release" / "govoplan_release" / "publisher.py").read_text() + + self.assertIn("candidate_catalog_version_issues(candidate_payload)", publisher) + self.assertIn("trusted_keys_from_keyring(\n target_keyring_payload", publisher) + self.assertIn("trusted_keys=publication_trust", publisher) + self.assertNotIn("trusted_keys=candidate_keys", publisher) + self.assertIn("write_module_directory(", publisher) + self.assertNotIn("shutil.copytree(candidate_modules", publisher) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_version_alignment.py b/tests/test_version_alignment.py new file mode 100644 index 0000000..221f996 --- /dev/null +++ b/tests/test_version_alignment.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile +import unittest + + +META_ROOT = Path(__file__).resolve().parents[1] +RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release" +if str(RELEASE_TOOLS_ROOT) not in sys.path: + sys.path.insert(0, str(RELEASE_TOOLS_ROOT)) + +from govoplan_release.version_alignment import ( # noqa: E402 + candidate_catalog_version_issues, + release_composition_issues, + repository_version_issues, + selected_repository_version_issues, +) +from govoplan_release.model import RepositorySnapshot, RepositorySpec, VersionSnapshot # noqa: E402 +from govoplan_release.selective_planner import build_unit # noqa: E402 +from govoplan_release.selective_catalog import enforce_selected_version_alignment # noqa: E402 + + +class VersionAlignmentTests(unittest.TestCase): + def test_candidate_catalog_refs_and_selected_units_must_match_versions(self) -> None: + payload = { + "core_release": { + "version": "1.2.3", + "python_ref": "govoplan-core @ git+ssh://git@example.test/acme/govoplan-core.git@v1.2.3", + }, + "modules": [ + { + "module_id": "files", + "version": "1.2.3", + "python_ref": "govoplan-files @ git+ssh://git@example.test/acme/govoplan-files.git@v1.2.4", + "webui_package": "@govoplan/files-webui", + "webui_ref": "git+ssh://git@example.test/acme/govoplan-files.git#v1.2.3", + } + ], + "release": { + "selected_units": [ + {"repo": "govoplan-files", "version": "1.2.5", "tag": "v1.2.6"}, + ] + }, + } + + issues = candidate_catalog_version_issues(payload) + + messages = {issue.message for issue in issues} + self.assertIn("Python ref tag must match the catalog entry version", messages) + self.assertIn("selected-unit tag must match its version", messages) + self.assertIn("selected-unit version must match the catalog entry", messages) + + def test_catalog_candidate_gate_rejects_selected_repo_drift(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) + repo = workspace / "govoplan-files" + (repo / "webui").mkdir(parents=True) + (repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n') + (repo / "webui" / "package.json").write_text('{"version":"1.2.4"}\n') + + with self.assertRaisesRegex(ValueError, "Version alignment gate failed"): + enforce_selected_version_alignment( + repo_versions={"govoplan-files": "1.2.5"}, + workspace=workspace, + ) + + def test_catalog_candidate_gate_rejects_requested_version_that_differs_from_source(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) + repo = workspace / "govoplan-files" + (repo / "webui").mkdir(parents=True) + (repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n') + (repo / "webui" / "package.json").write_text('{"version":"1.2.3"}\n') + + with self.assertRaisesRegex(ValueError, "source version must match the requested release version"): + enforce_selected_version_alignment( + repo_versions={"govoplan-files": "1.2.4"}, + workspace=workspace, + ) + + def test_selected_repository_gate_reports_missing_version_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) + (workspace / "govoplan-files").mkdir() + + issues = selected_repository_version_issues( + repo_versions={"govoplan-files": "1.2.3"}, + workspace=workspace, + ) + + self.assertEqual(1, len(issues)) + self.assertEqual("repository has no version metadata", issues[0].message) + + def test_selective_release_unit_is_blocked_by_version_drift(self) -> None: + repo = RepositorySnapshot( + spec=RepositorySpec( + name="govoplan-example", + category="module", + subtype="domain", + remote="git@example.test:acme/govoplan-example.git", + path="govoplan-example", + ), + absolute_path="/tmp/govoplan-example", + exists=True, + is_git=True, + has_head=True, + versions=VersionSnapshot(pyproject="1.2.3", webui_package="1.2.4", manifests=("1.2.3",)), + ) + + unit = build_unit(repo, target_version="1.2.5", contracts=None) + + self.assertEqual("blocked", unit.status) + self.assertIn("version metadata is not aligned", unit.blockers[0]) + + def test_repository_versions_and_lockfiles_must_match(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "govoplan-example" + (root / "src" / "govoplan_example" / "backend").mkdir(parents=True) + (root / "webui").mkdir() + (root / "pyproject.toml").write_text('[project]\nname = "govoplan-example"\nversion = "1.2.3"\n') + (root / "package.json").write_text('{"name":"@govoplan/example","version":"1.2.3"}\n') + (root / "webui" / "package.json").write_text('{"name":"@govoplan/example-webui","version":"1.2.4"}\n') + (root / "webui" / "package-lock.json").write_text( + json.dumps({"packages": {"": {"version": "1.2.2"}}}) + ) + (root / "src" / "govoplan_example" / "backend" / "manifest.py").write_text( + 'manifest = ModuleManifest(\n version="1.2.3",\n)\n' + ) + + issues = repository_version_issues(root) + + self.assertEqual(2, len(issues)) + self.assertEqual({"webui/package.json", "webui/package-lock.json"}, {issue.source for issue in issues}) + + def test_public_runtime_version_must_match_package_metadata(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "govoplan-example" + package = root / "src" / "govoplan_example" + package.mkdir(parents=True) + (root / "pyproject.toml").write_text( + '[project]\nname = "govoplan-example"\nversion = "1.2.3"\n' + ) + (package / "__init__.py").write_text('__version__ = "1.2.2"\n') + + issues = repository_version_issues(root) + + self.assertEqual(1, len(issues)) + self.assertEqual("package __init__.py 1", issues[0].source) + self.assertEqual("1.2.3", issues[0].expected) + self.assertEqual("1.2.2", issues[0].actual) + + def test_release_backend_and_frontend_refs_must_match(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + meta = Path(tmp) / "govoplan" + core = Path(tmp) / "govoplan-core" + meta.mkdir() + (core / "webui").mkdir(parents=True) + (meta / "requirements-release.txt").write_text( + "govoplan-example @ git+ssh://git@example.test/acme/govoplan-example.git@v1.2.3\n" + ) + (core / "webui" / "package.release.json").write_text( + json.dumps( + { + "dependencies": { + "@govoplan/example-webui": ( + "git+ssh://git@example.test/acme/govoplan-example.git#v1.2.4" + ) + } + } + ) + ) + + issues = release_composition_issues(meta, core_root=core) + + self.assertEqual(1, len(issues)) + self.assertEqual("1.2.3", issues[0].expected) + self.assertEqual("1.2.4", issues[0].actual) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/checks/check-release-integration.sh b/tools/checks/check-release-integration.sh index 73cba4b..216d5aa 100644 --- a/tools/checks/check-release-integration.sh +++ b/tools/checks/check-release-integration.sh @@ -92,6 +92,7 @@ install_cloned_webui_dependencies() { } run_step "Validate release refs and installed package metadata" +"$PYTHON" "$META_ROOT/tools/checks/check-version-alignment.py" --release-composition "$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact "$PYTHON" - <<'PY' from __future__ import annotations @@ -208,6 +209,12 @@ snapshot = registry.validate() print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests)) PY +run_step "Generate release dependency provenance" +"$PYTHON" "$META_ROOT/tools/release/generate-release-sbom.py" \ + --python "$PYTHON" \ + --core-root "$ROOT" \ + --output "$WORK_ROOT/govoplan-sbom.cdx.json" + run_step "Run SQLite migration smoke for release modules" "$PYTHON" - <<'PY' from __future__ import annotations diff --git a/tools/checks/check-version-alignment.py b/tools/checks/check-version-alignment.py new file mode 100644 index 0000000..b12d5ec --- /dev/null +++ b/tools/checks/check-version-alignment.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Fail when backend, manifest, frontend, lock, or release-ref versions drift.""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json +from pathlib import Path +import sys + + +META_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(META_ROOT / "tools" / "release")) + +from govoplan_release.version_alignment import ( # noqa: E402 + release_composition_issues, + repository_version_issues, + selected_repository_version_issues, +) +from govoplan_release.workspace import ( # noqa: E402 + load_repository_specs, + resolve_repo_path, + resolve_workspace_root, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace-root", type=Path, default=None) + parser.add_argument("--repo", action="append", default=[], help="Repository name to check; may be repeated.") + parser.add_argument( + "--repo-version", + action="append", + default=[], + metavar="REPO=VERSION", + help="Require a repository's aligned source metadata to equal VERSION; may be repeated.", + ) + parser.add_argument( + "--release-composition", + action="store_true", + help="Also compare tagged backend and WebUI references in the release composition.", + ) + parser.add_argument( + "--source-metadata-only", + action="store_true", + help="Skip lockfile checks for the pre-commit source gate; never use for publication.", + ) + parser.add_argument("--json", action="store_true", help="Print machine-readable output.") + args = parser.parse_args() + + workspace_root = resolve_workspace_root(args.workspace_root) + expected_versions = _parse_repo_versions(parser, args.repo_version) + selected = {*args.repo, *expected_versions} + known = {spec.name for spec in load_repository_specs()} + unknown = sorted(selected - known) + if unknown: + parser.error(f"unknown repository name(s): {', '.join(unknown)}") + + checked: list[str] = [] + issues = list( + selected_repository_version_issues( + repo_versions=expected_versions, + workspace=workspace_root, + include_lockfiles=not args.source_metadata_only, + ) + ) + for spec in load_repository_specs(): + if selected and spec.name not in selected: + continue + repo_path = resolve_repo_path(spec, workspace_root) + if not repo_path.exists(): + continue + checked.append(spec.name) + if spec.name in expected_versions: + continue + issues.extend( + repository_version_issues( + repo_path, + include_lockfiles=not args.source_metadata_only, + ) + ) + + if args.release_composition: + issues.extend( + release_composition_issues( + META_ROOT, + core_root=workspace_root / "govoplan-core", + ) + ) + + payload = { + "workspace_root": str(workspace_root), + "repositories_checked": checked, + "release_composition_checked": args.release_composition, + "issues": [asdict(issue) for issue in issues], + } + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + elif issues: + print("Version alignment failed:", file=sys.stderr) + for issue in issues: + print( + f"- {issue.repo}: {issue.source}: {issue.actual!r}; expected {issue.expected!r} " + f"({issue.message})", + file=sys.stderr, + ) + else: + suffix = " including release composition" if args.release_composition else "" + print(f"Version alignment passed for {len(checked)} repositories{suffix}.") + return 1 if issues else 0 + + +def _parse_repo_versions(parser: argparse.ArgumentParser, values: list[str]) -> dict[str, str]: + result: dict[str, str] = {} + for value in values: + if "=" not in value: + parser.error(f"--repo-version must use REPO=VERSION: {value}") + repo, version = (item.strip() for item in value.split("=", 1)) + version = version.removeprefix("v") + if not repo or not version: + parser.error(f"--repo-version must use REPO=VERSION: {value}") + if repo in result and result[repo] != version: + parser.error(f"conflicting versions requested for {repo}") + result[repo] = version + return result + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/generate-release-catalog.py b/tools/release/generate-release-catalog.py index a15575a..8466729 100644 --- a/tools/release/generate-release-catalog.py +++ b/tools/release/generate-release-catalog.py @@ -19,9 +19,11 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey META_ROOT = Path(__file__).resolve().parents[2] CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve() sys.path.insert(0, str(CORE_ROOT / "src")) +sys.path.insert(0, str(META_ROOT / "tools" / "release")) from govoplan_core.core.modules import ModuleManifest # noqa: E402 from govoplan_core.server.registry import available_module_manifests # noqa: E402 +from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402 GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas" @@ -196,6 +198,19 @@ def main() -> int: args = parser.parse_args() version = args.version.removeprefix("v") + version_issues = selected_repository_version_issues( + repo_versions={ + "govoplan-core": version, + **{module.repo: version for module in CATALOG_MODULES}, + }, + workspace=CORE_ROOT.parent, + ) + if version_issues: + details = "; ".join( + f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})" + for issue in version_issues + ) + parser.error(f"version alignment gate failed: {details}") tag = f"v{version}" generated_at = datetime.now(tz=UTC) sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M")) diff --git a/tools/release/generate-release-lock.sh b/tools/release/generate-release-lock.sh index 6f27a95..7ef6384 100644 --- a/tools/release/generate-release-lock.sh +++ b/tools/release/generate-release-lock.sh @@ -17,6 +17,10 @@ the regenerated release lockfile. Options: --npm npm executable to use. --core-root govoplan-core checkout. Defaults to ../govoplan-core. + --local-git-repo + Resolve that repository's release tag from its local Git + object store while preserving the published Git URL in + the lockfile. May be repeated. -h, --help Show this help. USAGE } @@ -32,6 +36,7 @@ CORE_ROOT="$(cd "$CORE_ROOT" && pwd)" WEBUI="$CORE_ROOT/webui" NPM_BIN="${NPM:-}" NODE_BIN="${NODE:-}" +LOCAL_GIT_REPOS=() if [[ -z "$NPM_BIN" ]]; then if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then @@ -61,6 +66,11 @@ while [[ $# -gt 0 ]]; do WEBUI="$CORE_ROOT/webui" shift 2 ;; + --local-git-repo) + [[ $# -ge 2 ]] || fail "missing value for $1" + LOCAL_GIT_REPOS+=("$(cd "$2" && pwd)") + shift 2 + ;; -h|--help) usage exit 0 @@ -90,9 +100,27 @@ fi echo "Generating release lockfile from $WEBUI/package.release.json" echo "Temporary workspace: $TMP_DIR" + +GIT_ENV=(env) +git_config_count=0 +for local_repo in "${LOCAL_GIT_REPOS[@]}"; do + [[ -d "$local_repo/.git" ]] || fail "local Git release source is not a repository: $local_repo" + remote_url="$(git -C "$local_repo" remote get-url origin)" + remote_url="${remote_url#git+}" + if [[ "$remote_url" != *://* && "$remote_url" =~ ^([^@]+@)?([^:]+):(.+)$ ]]; then + remote_url="ssh://${BASH_REMATCH[1]}${BASH_REMATCH[2]}/${BASH_REMATCH[3]}" + fi + GIT_ENV+=( + "GIT_CONFIG_KEY_${git_config_count}=url.file://$local_repo/.insteadOf" + "GIT_CONFIG_VALUE_${git_config_count}=$remote_url" + ) + git_config_count=$((git_config_count + 1)) +done +GIT_ENV+=("GIT_CONFIG_COUNT=$git_config_count") + ( cd "$TMP_DIR" - PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts + "${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts mapfile -t GIT_PACKAGES < <( PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE' const fs = require("fs"); @@ -108,7 +136,7 @@ NODE ) if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}" - PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}" + "${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}" fi ) diff --git a/tools/release/govoplan_release/git_state.py b/tools/release/govoplan_release/git_state.py index 88ded78..337752c 100644 --- a/tools/release/govoplan_release/git_state.py +++ b/tools/release/govoplan_release/git_state.py @@ -99,6 +99,7 @@ def collect_versions(path: Path) -> VersionSnapshot: package=read_json_version(path / "package.json"), webui_package=read_json_version(path / "webui" / "package.json"), manifests=read_manifest_versions(path), + package_inits=read_package_init_versions(path), ) @@ -137,6 +138,21 @@ def read_manifest_versions(path: Path) -> tuple[str, ...]: return tuple(versions) +def read_package_init_versions(path: Path) -> tuple[str, ...]: + """Read public runtime versions declared by top-level Python packages.""" + + src = path / "src" + if not src.exists(): + return () + versions: list[str] = [] + for package_init in sorted(src.glob("*/__init__.py")): + text = package_init.read_text(encoding="utf-8") + match = re.search(r'(?m)^__version__\s*=\s*["\']([^"\']+)["\']', text) + if match is not None: + versions.append(match.group(1)) + return tuple(versions) + + def git_text(path: Path, *args: str, timeout: int = 10) -> str: result = git(path, *args, timeout=timeout) return result.stdout.strip() if result.returncode == 0 else "" diff --git a/tools/release/govoplan_release/model.py b/tools/release/govoplan_release/model.py index 793aad2..e5f655a 100644 --- a/tools/release/govoplan_release/model.py +++ b/tools/release/govoplan_release/model.py @@ -26,10 +26,17 @@ class VersionSnapshot: package: str | None = None webui_package: str | None = None manifests: tuple[str, ...] = () + package_inits: tuple[str, ...] = () @property def primary(self) -> str | None: - return self.pyproject or self.package or self.webui_package or (self.manifests[0] if self.manifests else None) + return ( + self.pyproject + or self.package + or self.webui_package + or (self.manifests[0] if self.manifests else None) + or (self.package_inits[0] if self.package_inits else None) + ) @dataclass(frozen=True, slots=True) diff --git a/tools/release/govoplan_release/publisher.py b/tools/release/govoplan_release/publisher.py index 850d848..e9c3869 100644 --- a/tools/release/govoplan_release/publisher.py +++ b/tools/release/govoplan_release/publisher.py @@ -13,7 +13,9 @@ from govoplan_core.core.module_package_catalog import validate_module_package_ca from .catalog import canonical_hash from .model import CatalogPublishResult, CatalogPublishStep +from .module_directory import write_module_directory from .selective_catalog import trusted_keys_from_keyring +from .version_alignment import candidate_catalog_version_issues from .workspace import DEFAULT_WORKSPACE_ROOT, resolve_workspace_root, website_root @@ -41,7 +43,6 @@ def publish_catalog_candidate( resolved_web_root = Path(web_root).expanduser().resolve() if web_root is not None else website_root(workspace) candidate_catalog = candidate_root / "channels" / f"{channel}.json" candidate_keyring = candidate_root / "keyring.json" - candidate_modules = candidate_root / "modules" target_catalog = resolved_web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json" target_keyring = resolved_web_root / "public" / "catalogs" / "v1" / "keyring.json" target_modules = resolved_web_root / "public" / "catalogs" / "v1" / "modules" @@ -72,11 +73,36 @@ def publish_catalog_candidate( validation_warnings: tuple[str, ...] = () if candidate_payload is not None and keyring_payload is not None: + version_issues = candidate_catalog_version_issues(candidate_payload) + blockers.extend( + f"candidate version alignment: {issue.source}: {issue.actual!r}; " + f"expected {issue.expected!r} ({issue.message})" + for issue in version_issues + ) + candidate_keys = trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {}) + target_keyring_payload = read_json(target_keyring) if target_keyring.exists() else None + publication_trust = trusted_keys_from_keyring( + target_keyring_payload if isinstance(target_keyring_payload, dict) else {} + ) + if not publication_trust: + blockers.append( + "publication trust anchor is missing; bootstrap a trusted website keyring through a separate reviewed process" + ) + for key_id in sorted(set(candidate_keys) & set(publication_trust)): + if candidate_keys[key_id] != publication_trust[key_id]: + blockers.append(f"candidate keyring changes the public key for trusted key id {key_id!r}") + if candidate_keyring_hash != target_keyring_hash_before: + release = candidate_payload.get("release") if isinstance(candidate_payload, dict) else None + signed_keyring_hash = release.get("keyring_sha256") if isinstance(release, dict) else None + if signed_keyring_hash != candidate_keyring_hash: + blockers.append( + "candidate keyring differs from the publication trust anchor without a matching signed keyring_sha256" + ) validation = validate_module_package_catalog( candidate_catalog, require_trusted=True, approved_channels=(channel,), - trusted_keys=trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {}), + trusted_keys=publication_trust, ) validation_valid = validation.get("valid") is True validation_error = str(validation["error"]) if validation.get("error") else None @@ -115,14 +141,12 @@ def publish_catalog_candidate( steps.append( CatalogPublishStep( id="copy", - title="Copy candidate catalog, keyring, and module directory", + title="Copy candidate catalog/keyring and derive the module directory", detail=f"{candidate_root} -> {resolved_web_root / 'public' / 'catalogs' / 'v1'}", mutating=True, status="planned" if not apply else "blocked" if blockers else "done", ) ) - if not candidate_modules.exists(): - notes.append("Candidate has no module-directory tree; only catalog and keyring will be copied.") if build_web: steps.append( CatalogPublishStep( @@ -224,10 +248,15 @@ def publish_catalog_candidate( target_catalog.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(candidate_catalog, target_catalog) shutil.copy2(candidate_keyring, target_keyring) - if candidate_modules.exists(): - if target_modules.exists(): - shutil.rmtree(target_modules) - shutil.copytree(candidate_modules, target_modules) + if target_modules.exists(): + shutil.rmtree(target_modules) + write_module_directory( + catalog_payload=candidate_payload, + keyring_payload=keyring_payload, + output_root=target_catalog.parent.parent, + channel=channel, + public_base_url=catalog_public_base_url(candidate_payload), + ) completed_steps = [replace_status(step, "done") if step.id == "copy" else step for step in steps] if build_web: @@ -282,6 +311,14 @@ def publish_catalog_candidate( ) +def catalog_public_base_url(payload: dict[str, object]) -> str: + release = payload.get("release") + catalog_url = release.get("catalog_url") if isinstance(release, dict) else None + if isinstance(catalog_url, str) and "/catalogs/" in catalog_url: + return catalog_url.split("/catalogs/", 1)[0] + return "https://govoplan.add-ideas.de" + + def result( *, status: str, diff --git a/tools/release/govoplan_release/selective_catalog.py b/tools/release/govoplan_release/selective_catalog.py index ae1c74e..54fbaa6 100644 --- a/tools/release/govoplan_release/selective_catalog.py +++ b/tools/release/govoplan_release/selective_catalog.py @@ -19,6 +19,7 @@ from .contracts import collect_contracts from .git_state import collect_repository_snapshot from .model import CatalogEntryChange, SelectiveCatalogCandidate from .module_directory import write_module_directory +from .version_alignment import selected_repository_version_issues from .workspace import load_repository_specs, resolve_workspace_root, website_root GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas" @@ -46,6 +47,7 @@ def build_selective_catalog_candidate( raise ValueError("At least one signing key is required.") workspace = resolve_workspace_root(workspace_root) + enforce_selected_version_alignment(repo_versions=repo_versions, workspace=workspace) web_root = website_root(workspace) source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel) payload = read_catalog(source_catalog) @@ -75,22 +77,24 @@ def build_selective_catalog_candidate( repository_base=repository_base.rstrip("/"), ) - candidate.pop("signature", None) - candidate.pop("signatures", None) - candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys] - output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at) catalog_path = output_root / "channels" / f"{channel}.json" keyring_path = output_root / "keyring.json" summary_path = output_root / "summary.json" if write_summary else None - catalog_path.parent.mkdir(parents=True, exist_ok=True) - catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8") - keyring_payload = keyring_payload_for_candidate( existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json", signing_keys=parsed_keys, generated_at=generated_at, ) + release["keyring_sha256"] = canonical_hash(keyring_payload) + + candidate.pop("signature", None) + candidate.pop("signatures", None) + candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys] + + catalog_path.parent.mkdir(parents=True, exist_ok=True) + catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8") + keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") module_directory_files = write_module_directory( catalog_payload=candidate, @@ -159,6 +163,18 @@ def build_selective_catalog_candidate( return result +def enforce_selected_version_alignment(*, repo_versions: dict[str, str], workspace: Path) -> None: + failures = [ + f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})" + for issue in selected_repository_version_issues( + repo_versions=repo_versions, + workspace=workspace, + ) + ] + if failures: + raise ValueError("Version alignment gate failed: " + "; ".join(failures)) + + def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str: if base_catalog is not None: value = str(base_catalog) diff --git a/tools/release/govoplan_release/selective_planner.py b/tools/release/govoplan_release/selective_planner.py index b870eb1..fef333a 100644 --- a/tools/release/govoplan_release/selective_planner.py +++ b/tools/release/govoplan_release/selective_planner.py @@ -76,6 +76,19 @@ def build_unit(repo: RepositorySnapshot, *, target_version: str | None, contract blockers.append(f"repository is behind {repo.upstream}") if repo.exists and repo.is_git and not repo.has_head: blockers.append("repository has no initial commit") + local_versions = tuple( + value + for value in ( + repo.versions.pyproject, + repo.versions.package, + repo.versions.webui_package, + *repo.versions.manifests, + *repo.versions.package_inits, + ) + if value + ) + if len(set(local_versions)) > 1: + blockers.append("backend, manifest, and frontend version metadata is not aligned") if repo.dirty: warnings.append("uncommitted changes will need review before release") if repo.ahead: diff --git a/tools/release/govoplan_release/version_alignment.py b/tools/release/govoplan_release/version_alignment.py new file mode 100644 index 0000000..c3dcbe2 --- /dev/null +++ b/tools/release/govoplan_release/version_alignment.py @@ -0,0 +1,572 @@ +"""Version-alignment checks for independently released GovOPlaN packages.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +import re +import subprocess + +from .git_state import collect_versions +from .workspace import load_repository_specs, resolve_repo_path + + +_PYTHON_RELEASE_REF = re.compile( + r"^(?Pgovoplan-[a-z0-9-]+)\s+@\s+git\+ssh://.+/" + r"(?Pgovoplan-[a-z0-9-]+)\.git@v(?P[^\s;]+)$" +) +_WEBUI_RELEASE_REF = re.compile(r"/(?Pgovoplan-[a-z0-9-]+)\.git#v(?P[^#\s]+)$") +_CATALOG_PYTHON_REF = re.compile(r"/(?Pgovoplan-[a-z0-9-]+)\.git@v(?P[^\s;]+)$") +_CATALOG_WEBUI_REF = re.compile(r"/(?Pgovoplan-[a-z0-9-]+)\.git#v(?P[^#\s]+)$") + + +@dataclass(frozen=True, slots=True) +class VersionAlignmentIssue: + repo: str + source: str + expected: str + actual: str + message: str + + +def repository_version_issues( + repo_path: Path, + *, + expected_version: str | None = None, + include_lockfiles: bool = True, +) -> tuple[VersionAlignmentIssue, ...]: + """Compare every version-bearing file in one source repository.""" + + versions = collect_versions(repo_path) + declared = { + "pyproject.toml": versions.pyproject, + "package.json": versions.package, + "webui/package.json": versions.webui_package, + "webui/package.release.json": _json_version(repo_path / "webui" / "package.release.json") + if (repo_path / "webui" / "package.release.json").exists() + else None, + } + for index, version in enumerate(versions.manifests, start=1): + declared[f"module manifest {index}"] = version + for index, version in enumerate(versions.package_inits, start=1): + declared[f"package __init__.py {index}"] = version + + canonical_source, canonical_version = next( + ((source, version) for source, version in declared.items() if version), + (None, None), + ) + repo = repo_path.name + if canonical_source is None or canonical_version is None: + if expected_version is None: + return () + return ( + VersionAlignmentIssue( + repo=repo, + source="repository", + expected=expected_version.removeprefix("v"), + actual="", + message="repository has no version metadata", + ), + ) + + issues = [ + VersionAlignmentIssue( + repo=repo, + source=source, + expected=canonical_version, + actual=version, + message=f"{source} must match {canonical_source}", + ) + for source, version in declared.items() + if version is not None and version != canonical_version + ] + + if expected_version is not None and canonical_version.removeprefix("v") != expected_version.removeprefix("v"): + issues.append( + VersionAlignmentIssue( + repo=repo, + source=canonical_source, + expected=expected_version.removeprefix("v"), + actual=canonical_version, + message="source version must match the requested release version", + ) + ) + + if include_lockfiles: + issues.extend( + _lockfile_issues( + repo=repo, + package_path=repo_path / "package.json", + lock_path=repo_path / "package-lock.json", + ) + ) + issues.extend( + _lockfile_issues( + repo=repo, + package_path=repo_path / "webui" / "package.json", + lock_path=repo_path / "webui" / "package-lock.json", + ) + ) + issues.extend( + _lockfile_issues( + repo=repo, + package_path=repo_path / "webui" / "package.release.json", + lock_path=repo_path / "webui" / "package-lock.release.json", + ) + ) + return tuple(issues) + + +def selected_repository_version_issues( + *, + repo_versions: dict[str, str], + workspace: Path, + include_lockfiles: bool = True, +) -> tuple[VersionAlignmentIssue, ...]: + """Validate registered selected worktrees against explicit release versions.""" + + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + issues: list[VersionAlignmentIssue] = [] + for repo, expected_version in sorted(repo_versions.items()): + spec = specs.get(repo) + if spec is None: + issues.append( + VersionAlignmentIssue( + repo=repo, + source="repository", + expected=expected_version.removeprefix("v"), + actual="", + message="repository is not registered", + ) + ) + continue + repo_path = resolve_repo_path(spec, workspace) + if not repo_path.exists(): + issues.append( + VersionAlignmentIssue( + repo=repo, + source="repository", + expected=expected_version.removeprefix("v"), + actual="", + message="repository path is missing", + ) + ) + continue + issues.extend( + repository_version_issues( + repo_path, + expected_version=expected_version, + include_lockfiles=include_lockfiles, + ) + ) + return tuple(issues) + + +def release_composition_issues(meta_root: Path, *, core_root: Path) -> tuple[VersionAlignmentIssue, ...]: + """Require backend and WebUI release references for a module to agree.""" + + python_refs = _python_release_refs(meta_root / "requirements-release.txt") + webui_refs = _webui_release_refs(core_root / "webui" / "package.release.json") + issues: list[VersionAlignmentIssue] = [] + + for repo in sorted(set(python_refs) & set(webui_refs)): + python_version = python_refs[repo] + webui_version = webui_refs[repo] + if python_version == webui_version: + continue + issues.append( + VersionAlignmentIssue( + repo=repo, + source="webui/package.release.json", + expected=python_version, + actual=webui_version, + message="frontend release ref must match the backend release ref", + ) + ) + return tuple(issues) + + +def candidate_catalog_version_issues(payload: object) -> tuple[VersionAlignmentIssue, ...]: + """Validate versions, refs, and selected-unit provenance inside a catalog.""" + + if not isinstance(payload, dict): + return ( + VersionAlignmentIssue( + repo="catalog", + source="catalog", + expected="object", + actual=type(payload).__name__, + message="candidate catalog must be a JSON object", + ), + ) + + issues: list[VersionAlignmentIssue] = [] + represented: dict[str, str] = {} + core_release = payload.get("core_release") + if isinstance(core_release, dict): + issues.extend(_catalog_entry_issues(core_release, source="core_release", represented=represented)) + else: + issues.append(_catalog_shape_issue("core_release", "candidate catalog has no core release")) + + modules = payload.get("modules") + if not isinstance(modules, list): + issues.append(_catalog_shape_issue("modules", "candidate catalog has no modules list")) + else: + for index, entry in enumerate(modules): + if not isinstance(entry, dict): + issues.append(_catalog_shape_issue(f"modules[{index}]", "module entry must be an object")) + continue + issues.extend( + _catalog_entry_issues( + entry, + source=f"modules[{index}]", + represented=represented, + ) + ) + + release = payload.get("release") + if isinstance(release, dict): + issues.extend(_catalog_release_issues(release, represented=represented)) + return tuple(issues) + + +def _catalog_entry_issues( + entry: dict[str, object], + *, + source: str, + represented: dict[str, str], +) -> list[VersionAlignmentIssue]: + issues: list[VersionAlignmentIssue] = [] + version = entry.get("version") + normalized_version = version.removeprefix("v") if isinstance(version, str) and version else None + if normalized_version is None: + issues.append(_catalog_shape_issue(f"{source}.version", "catalog entry has no version")) + + python_ref = entry.get("python_ref") + python_match = _CATALOG_PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None + if python_match is None: + issues.append(_catalog_shape_issue(f"{source}.python_ref", "Python ref must end in a version tag")) + repo = str(entry.get("python_package") or entry.get("module_id") or source) + else: + repo = python_match.group("repo") + ref_version = python_match.group("version").removeprefix("v") + if normalized_version is not None and ref_version != normalized_version: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{source}.python_ref", + expected=normalized_version, + actual=ref_version, + message="Python ref tag must match the catalog entry version", + ) + ) + if normalized_version is not None: + previous = represented.setdefault(repo, normalized_version) + if previous != normalized_version: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=source, + expected=previous, + actual=normalized_version, + message="repository appears with conflicting catalog versions", + ) + ) + + webui_package = entry.get("webui_package") + webui_ref = entry.get("webui_ref") + if bool(webui_package) != bool(webui_ref): + issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI package and ref must be declared together")) + if isinstance(webui_ref, str): + webui_match = _CATALOG_WEBUI_REF.search(webui_ref) + if webui_match is None: + issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI ref must end in a version tag")) + else: + webui_repo = webui_match.group("repo") + webui_version = webui_match.group("version").removeprefix("v") + if python_match is not None and webui_repo != repo: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{source}.webui_ref", + expected=repo, + actual=webui_repo, + message="backend and WebUI refs must use the same repository", + ) + ) + if normalized_version is not None and webui_version != normalized_version: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{source}.webui_ref", + expected=normalized_version, + actual=webui_version, + message="WebUI ref tag must match the catalog entry version", + ) + ) + return issues + + +def _catalog_release_issues( + release: dict[str, object], + *, + represented: dict[str, str], +) -> list[VersionAlignmentIssue]: + issues: list[VersionAlignmentIssue] = [] + version = release.get("version") + tag = release.get("tag") + if isinstance(version, str) and isinstance(tag, str) and tag != f"v{version.removeprefix('v')}": + issues.append( + VersionAlignmentIssue( + repo="govoplan-core", + source="release.tag", + expected=f"v{version.removeprefix('v')}", + actual=tag, + message="release tag must match release version", + ) + ) + + selected_units = release.get("selected_units") + if selected_units is None: + return issues + if not isinstance(selected_units, list): + issues.append(_catalog_shape_issue("release.selected_units", "selected units must be a list")) + return issues + seen: set[str] = set() + for index, unit in enumerate(selected_units): + if not isinstance(unit, dict): + issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit must be an object")) + continue + repo = unit.get("repo") + unit_version = unit.get("version") + unit_tag = unit.get("tag") + if not isinstance(repo, str) or not isinstance(unit_version, str) or not isinstance(unit_tag, str): + issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit requires repo, version, and tag")) + continue + normalized_version = unit_version.removeprefix("v") + if repo in seen: + issues.append(_catalog_shape_issue(f"release.selected_units[{index}].repo", "selected repository is duplicated")) + seen.add(repo) + if unit_tag != f"v{normalized_version}": + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"release.selected_units[{index}].tag", + expected=f"v{normalized_version}", + actual=unit_tag, + message="selected-unit tag must match its version", + ) + ) + represented_version = represented.get(repo) + if represented_version is None: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"release.selected_units[{index}]", + expected=normalized_version, + actual="", + message="selected repository is not represented in the catalog", + ) + ) + elif represented_version != normalized_version: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"release.selected_units[{index}].version", + expected=represented_version, + actual=normalized_version, + message="selected-unit version must match the catalog entry", + ) + ) + return issues + + +def _catalog_shape_issue(source: str, message: str) -> VersionAlignmentIssue: + return VersionAlignmentIssue( + repo="catalog", + source=source, + expected="valid value", + actual="", + message=message, + ) + + +def _lockfile_issues(*, repo: str, package_path: Path, lock_path: Path) -> list[VersionAlignmentIssue]: + if not package_path.exists() or not lock_path.exists(): + return [] + package_payload = _json_object(package_path) + lock_payload = _json_object(lock_path) + package_version = _payload_version(package_payload) + lock_version = _lockfile_root_version(lock_path) + issues: list[VersionAlignmentIssue] = [] + if package_version is None or lock_version is None or package_version == lock_version: + pass + else: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=str(lock_path.relative_to(package_path.parents[1] if package_path.parent.name == "webui" else package_path.parent)), + expected=package_version, + actual=lock_version, + message=f"lockfile root version must match {package_path.name}", + ) + ) + lock_packages = lock_payload.get("packages") + lock_root = lock_packages.get("") if isinstance(lock_packages, dict) else None + if isinstance(lock_root, dict): + for group in ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies"): + package_dependencies = package_payload.get(group) or {} + lock_dependencies = lock_root.get(group) or {} + if package_dependencies != lock_dependencies: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{lock_path.name}:{group}", + expected=json.dumps(package_dependencies, sort_keys=True), + actual=json.dumps(lock_dependencies, sort_keys=True), + message=f"lockfile root {group} must match {package_path.name}", + ) + ) + issues.extend( + _git_lock_resolution_issues( + repo=repo, + package_payload=package_payload, + lock_packages=lock_packages, + workspace=package_path.parents[2] if package_path.parent.name == "webui" else package_path.parent.parent, + lock_name=lock_path.name, + ) + ) + return issues + + +def _git_lock_resolution_issues( + *, + repo: str, + package_payload: dict[str, object], + lock_packages: dict[str, object], + workspace: Path, + lock_name: str, +) -> list[VersionAlignmentIssue]: + dependencies = package_payload.get("dependencies") + if not isinstance(dependencies, dict): + return [] + issues: list[VersionAlignmentIssue] = [] + for package_name, spec in dependencies.items(): + if not isinstance(package_name, str) or not isinstance(spec, str): + continue + match = _CATALOG_WEBUI_REF.search(spec) + if match is None: + continue + source_repo = match.group("repo") + version = match.group("version").removeprefix("v") + locked = lock_packages.get(f"node_modules/{package_name}") + if not isinstance(locked, dict): + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{lock_name}:{package_name}", + expected=version, + actual="", + message="release lock has no resolved GovOPlaN package", + ) + ) + continue + locked_version = locked.get("version") + if locked_version != version: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{lock_name}:{package_name}", + expected=version, + actual=str(locked_version or ""), + message="resolved package version must match its release ref", + ) + ) + expected_commit = _git_tag_commit(workspace / source_repo, f"v{version}") + resolved = locked.get("resolved") + resolved_commit = resolved.rsplit("#", 1)[-1] if isinstance(resolved, str) and "#" in resolved else None + if expected_commit is None or resolved_commit != expected_commit: + issues.append( + VersionAlignmentIssue( + repo=repo, + source=f"{lock_name}:{package_name}:resolved", + expected=expected_commit or f"local tag v{version}", + actual=resolved_commit or "", + message="resolved package commit must match the local release tag", + ) + ) + return issues + + +def _git_tag_commit(repo_path: Path, tag: str) -> str | None: + if not (repo_path / ".git").exists(): + return None + result = subprocess.run( + ["git", "-C", str(repo_path), "rev-list", "-n", "1", tag], + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=10, + ) + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def _json_version(path: Path) -> str | None: + value = _payload_version(_json_object(path)) + return value + + +def _payload_version(payload: dict[str, object]) -> str | None: + value = payload.get("version") + return value if isinstance(value, str) else None + + +def _json_object(path: Path) -> dict[str, object]: + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + + +def _lockfile_root_version(path: Path) -> str | None: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + return None + packages = payload.get("packages") + if isinstance(packages, dict): + root = packages.get("") + if isinstance(root, dict) and isinstance(root.get("version"), str): + return root["version"] + value = payload.get("version") + return value if isinstance(value, str) else None + + +def _python_release_refs(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + refs: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = _PYTHON_RELEASE_REF.match(line.strip()) + if match is None: + continue + repo = match.group("repo") + refs[repo] = match.group("version") + return refs + + +def _webui_release_refs(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + dependencies = payload.get("dependencies") if isinstance(payload, dict) else None + if not isinstance(dependencies, dict): + return {} + refs: dict[str, str] = {} + for spec in dependencies.values(): + if not isinstance(spec, str): + continue + match = _WEBUI_RELEASE_REF.search(spec) + if match is not None: + refs[match.group("repo")] = match.group("version") + return refs diff --git a/tools/release/publish-release-catalog.sh b/tools/release/publish-release-catalog.sh index bdab460..223b1a4 100644 --- a/tools/release/publish-release-catalog.sh +++ b/tools/release/publish-release-catalog.sh @@ -194,6 +194,7 @@ run() { } GEN_ARGS=( + env "GOVOPLAN_CORE_ROOT=$CORE_ROOT" "$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py" --version "$VERSION" --channel "$CHANNEL" diff --git a/tools/release/push-release-tag.sh b/tools/release/push-release-tag.sh index 60ac635..1a06b3b 100644 --- a/tools/release/push-release-tag.sh +++ b/tools/release/push-release-tag.sh @@ -52,28 +52,10 @@ Options: -h, --help Show this help. Repos: - Installable release packages: - govoplan-access - govoplan-admin - govoplan-tenancy - govoplan-organizations - govoplan-identity - govoplan-idm - govoplan-policy - govoplan-audit - govoplan-dashboard - govoplan-files - govoplan-mail - govoplan-campaign - govoplan-calendar - govoplan-ops - govoplan-core - - Roadmap/scaffold module repositories are tag-only until they have package - metadata: addresses, appointments, cases, connectors, dms, erp, - fit-connect, forms, identity-trust, ledger, notifications, payments, - portal, reporting, scheduling, search, tasks, templates, workflow, xoev, - xrechnung, and xta-osci. + Core, every registered release/module repository listed below in this script, + and the meta repository. Repositories with pyproject.toml are versioned and + alignment-gated dynamically; remaining roadmap/scaffold repositories are + committed and tagged without inventing package metadata. Examples: tools/release/push-release-tag.sh @@ -121,12 +103,14 @@ PACKAGE_MODULE_REPOS=( "$PARENT/govoplan-organizations" "$PARENT/govoplan-permits" "$PARENT/govoplan-policy" + "$PARENT/govoplan-poll" "$PARENT/govoplan-procurement" "$PARENT/govoplan-records" "$PARENT/govoplan-resources" "$PARENT/govoplan-risk-compliance" "$PARENT/govoplan-tenancy" "$PARENT/govoplan-transparency" + "$PARENT/govoplan-evaluation" ) TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-addresses" @@ -134,6 +118,7 @@ TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-cases" "$PARENT/govoplan-connectors" "$PARENT/govoplan-dms" + "$PARENT/govoplan-dist-lists" "$PARENT/govoplan-erp" "$PARENT/govoplan-fit-connect" "$PARENT/govoplan-forms" @@ -144,8 +129,10 @@ TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-portal" "$PARENT/govoplan-postbox" "$PARENT/govoplan-reporting" + "$PARENT/govoplan-rest" "$PARENT/govoplan-scheduling" "$PARENT/govoplan-search" + "$PARENT/govoplan-soap" "$PARENT/govoplan-tasks" "$PARENT/govoplan-templates" "$PARENT/govoplan-workflow" @@ -154,10 +141,12 @@ TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-xta-osci" ) MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}") -PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT") +SUPPORT_REPOS=("$META_ROOT") +PACKAGE_REPOS=() REPOS=( "${MODULE_REPOS[@]}" "$ROOT" + "${SUPPORT_REPOS[@]}" ) REMOTE="origin" @@ -379,6 +368,7 @@ if project_name != "govoplan-core": peers["@govoplan/core-webui"] = f"^{new_version}" path.write_text(json.dumps(data, indent=2) + "\n") PYCODE + synchronize_lockfile_root "$package_path" "${package_path%package.json}package-lock.json" done if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then @@ -401,9 +391,59 @@ for name, spec in list(dependencies.items()): dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec) path.write_text(json.dumps(data, indent=2) + "\n") PYCODE + synchronize_lockfile_root "$release_package_path" "$repo/webui/package-lock.release.json" fi } +synchronize_lockfile_root() { + local package_path="$1" + local lock_path="$2" + [[ -f "$package_path" && -f "$lock_path" ]] || return 0 + "$PYTHON" - "$package_path" "$lock_path" <<'PYCODE' +from __future__ import annotations + +import json +from pathlib import Path +import sys + +package_path = Path(sys.argv[1]) +lock_path = Path(sys.argv[2]) +package = json.loads(package_path.read_text()) +lock = json.loads(lock_path.read_text()) +version = package.get("version") +if not isinstance(version, str) or not version: + raise SystemExit(f"package version is missing from {package_path}") +lock["version"] = version +packages = lock.get("packages") +if isinstance(packages, dict) and isinstance(packages.get(""), dict): + packages[""]["version"] = version +lock_path.write_text(json.dumps(lock, indent=2) + "\n") +PYCODE +} + +update_release_requirements() { + local version="$1" + "$PYTHON" - "$META_ROOT/requirements-release.txt" "$version" <<'PYCODE' +from __future__ import annotations + +from pathlib import Path +import re +import sys + +path = Path(sys.argv[1]) +version = sys.argv[2] +text = path.read_text() +updated, count = re.subn( + r"(?m)^(govoplan-[a-z0-9-]+\s+@\s+git\+ssh://[^\s]+\.git)@v[^\s]+$", + rf"\1@v{version}", + text, +) +if count == 0: + raise SystemExit(f"no tagged GovOPlaN requirements found in {path}") +path.write_text(updated) +PYCODE +} + update_version_files() { local repo="$1" local version="$2" @@ -427,7 +467,30 @@ generate_release_lock() { fi [[ -x "$META_ROOT/tools/release/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $META_ROOT/tools/release/generate-release-lock.sh" - run "$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN" + local command=("$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN") + local repo + for repo in "${MODULE_REPOS[@]}"; do + command+=(--local-git-repo "$repo") + done + run "${command[@]}" +} + +run_version_alignment_gate() { + local mode="${1:-full}" + local command=( + "$PYTHON" + "$META_ROOT/tools/checks/check-version-alignment.py" + --workspace-root "$PARENT" + --release-composition + ) + if [[ "$mode" == "source" ]]; then + command+=(--source-metadata-only) + fi + local repo + for repo in "${PACKAGE_REPOS[@]}"; do + command+=(--repo-version "$(basename "$repo")=$TARGET_VERSION") + done + run "${command[@]}" } run_migration_release_audit() { @@ -694,6 +757,15 @@ for repo in "${REPOS[@]}"; do fi done +EFFECTIVE_TAG_ONLY_REPOS=() +for repo in "${REPOS[@]}"; do + if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then + PACKAGE_REPOS+=("$repo") + else + EFFECTIVE_TAG_ONLY_REPOS+=("$repo") + fi +done + if [[ -z "$TARGET_VERSION" ]]; then BASE_VERSION="${OLD_VERSIONS[$ROOT]}" for repo in "${PACKAGE_REPOS[@]}"; do @@ -744,9 +816,9 @@ echo " tag: $TAG" echo " remote: $REMOTE" echo " commit message: $COMMIT_MESSAGE" echo " package repos: ${#PACKAGE_REPOS[@]} versioned" -echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files" +echo " tag-only/support repos: ${#EFFECTIVE_TAG_ONLY_REPOS[@]} committed/tagged/pushed without package version files" if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then - echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed" + echo " release lock: resolve core webui/package-lock.release.json from local reviewed module tags" else echo " release lock: skipped" fi @@ -780,9 +852,21 @@ for repo in "${PACKAGE_REPOS[@]}"; do fi done +if [[ "$DRY_RUN" -eq 1 ]]; then + echo "Would update $META_ROOT/requirements-release.txt to $TAG" +else + update_release_requirements "$TARGET_VERSION" +fi + refresh_development_webui_lock -for repo in "${MODULE_REPOS[@]}"; do +# Source metadata, release refs, and lock roots must agree before creating any +# release commit or tag. A second gate below verifies the fully resolved release +# lock before any remote mutation. +run_version_alignment_gate source + +PRE_CORE_REPOS=("${MODULE_REPOS[@]}" "${SUPPORT_REPOS[@]}") +for repo in "${PRE_CORE_REPOS[@]}"; do run git -C "$repo" add -A if [[ "$DRY_RUN" -eq 0 ]]; then @@ -806,12 +890,13 @@ for repo in "${MODULE_REPOS[@]}"; do run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE" done -for repo in "${MODULE_REPOS[@]}"; do - run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG" -done - generate_release_lock +# npm resolves the new module tags from the local repositories. Nothing has +# been pushed yet; a failed lock or alignment check leaves only local, +# recoverable commits/tags. +run_version_alignment_gate + run git -C "$ROOT" add -A if [[ "$DRY_RUN" -eq 0 ]]; then @@ -822,6 +907,10 @@ fi run git -C "$ROOT" commit -m "$COMMIT_MESSAGE" run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE" + +for repo in "${PRE_CORE_REPOS[@]}"; do + run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG" +done run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG" if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then diff --git a/tools/release/release-doctor.py b/tools/release/release-doctor.py index be29476..26514c5 100644 --- a/tools/release/release-doctor.py +++ b/tools/release/release-doctor.py @@ -381,7 +381,7 @@ def repo_findings(repos: tuple[RepoState, ...], *, target_version: str | None, t findings.append( Finding( check_id="repo-version-mismatch", - severity="warning", + severity="blocker", title=f"{repo.name} has inconsistent local version files", detail=", ".join( value