diff --git a/.gitea/workflows/publish-developer-meta-package.yml b/.gitea/workflows/publish-developer-meta-package.yml new file mode 100644 index 0000000..28a620f --- /dev/null +++ b/.gitea/workflows/publish-developer-meta-package.yml @@ -0,0 +1,60 @@ +name: Developer Meta-package Release + +on: + push: + tags: + - "v*" + +jobs: + publish-package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Validate protected release tag and package version + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + python - <<'PY' + import fnmatch + import json + import os + from pathlib import Path + import subprocess + import tomllib + import urllib.request + + tag = os.environ["GITEA_REF_NAME"] + project = tomllib.loads(Path("packages/govoplan-meta/pyproject.toml").read_text(encoding="utf-8"))["project"] + if tag != f"v{project['version']}": + raise SystemExit("meta-package version does not match the release tag") + if subprocess.run(["git", "merge-base", "--is-ancestor", "HEAD", "origin/main"]).returncode: + raise SystemExit("release tag is not contained in main") + request = urllib.request.Request( + f"{os.environ['GITEA_API_URL']}/repos/{os.environ['GITEA_REPOSITORY']}/tag_protections", + headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + protections = json.load(response) + if not any(fnmatch.fnmatchcase(tag, item.get("name_pattern", "")) for item in protections): + raise SystemExit("release tag is not protected") + PY + - name: Build and publish developer package + env: + PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }} + PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }} + run: | + set -euo pipefail + test -n "$PACKAGE_USERNAME" + test -n "$PACKAGE_TOKEN" + python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0 + python -m build --wheel --outdir dist packages/govoplan-meta + python -m twine check dist/*.whl + TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \ + python -m twine upload --non-interactive \ + --repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \ + dist/*.whl diff --git a/.gitea/workflows/release-integration.yml b/.gitea/workflows/release-integration.yml index f42279a..05affe5 100644 --- a/.gitea/workflows/release-integration.yml +++ b/.gitea/workflows/release-integration.yml @@ -25,6 +25,12 @@ jobs: - name: Bootstrap GovOPlaN repositories working-directory: govoplan run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website + - name: Validate package publication contracts + working-directory: govoplan + run: | + python tools/repo/sync-module-package-workflows.py --check + python tools/release/generate-developer-meta-package.py --check + python -m unittest tests.test_module_package_workflows tests.test_package_registry_release - name: Install backend release integration dependencies working-directory: govoplan run: | diff --git a/.gitea/workflows/runtime-distribution.yml b/.gitea/workflows/runtime-distribution.yml index 6c1c483..c407577 100644 --- a/.gitea/workflows/runtime-distribution.yml +++ b/.gitea/workflows/runtime-distribution.yml @@ -101,12 +101,27 @@ jobs: run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website - name: Build release wheel roots and WebUI working-directory: govoplan + env: + VERSION: ${{ inputs.version }} + GOVOPLAN_PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }} + GOVOPLAN_PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }} run: | python -m venv .runtime-build - .runtime-build/bin/python -m pip install --upgrade pip wheel cryptography - mkdir -p runtime-output/local-wheels - .runtime-build/bin/python -m pip wheel --no-deps --wheel-dir runtime-output/local-wheels --requirement requirements-release.txt - bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui + .runtime-build/bin/python -m pip install --upgrade pip cryptography + .runtime-build/bin/python tools/release/generate-release-package-set.py \ + --version "$VERSION" \ + --output runtime-output/release-packages.json + .runtime-build/bin/python tools/release/resolve-package-artifacts.py \ + --package-set runtime-output/release-packages.json \ + --wheelhouse runtime-output/local-wheels \ + --webui-packages runtime-output/webui-packages \ + --lock-output runtime-output/package-artifacts.lock.json \ + --requirements-output runtime-output/requirements-release.packages.txt \ + --python .runtime-build/bin/python + PYTHON=.runtime-build/bin/python \ + GOVOPLAN_WEBUI_PACKAGE_LOCK="$PWD/runtime-output/package-artifacts.lock.json" \ + GOVOPLAN_WEBUI_PACKAGE_DIR="$PWD/runtime-output/webui-packages" \ + bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui npm --prefix ../govoplan-core/webui run build .runtime-build/bin/python tools/release/prepare-runtime-context.py \ --wheelhouse runtime-output/local-wheels \ @@ -264,6 +279,7 @@ jobs: --web-metadata runtime-output/web-metadata.json \ --deployer runtime-output/govoplan-deploy.pyz \ --deployer-url "$ARTIFACT_BASE/govoplan-deploy.pyz" \ + --package-lock runtime-output/package-artifacts.lock.json \ --artifact-base-url "$ARTIFACT_BASE" \ --source-commit "$SOURCE_COMMIT" \ --version "$VERSION" \ @@ -361,6 +377,9 @@ jobs: --asset runtime-output/distribution-manifest.json.sha256 \ --asset runtime-output/distribution-keyring.json \ --asset runtime-output/context-amd64/composition.json \ + --asset runtime-output/release-packages.json \ + --asset runtime-output/package-artifacts.lock.json \ + --asset runtime-output/requirements-release.packages.txt \ --asset runtime-output/evidence/api-sbom.cdx.json \ --asset runtime-output/evidence/web-sbom.cdx.json \ --asset runtime-output/evidence/api-provenance.json \ diff --git a/.gitignore b/.gitignore index 8c5260a..b474ab4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ tools/release/runtime/* !tools/release/runtime/Dockerfile.web !tools/release/runtime/nginx.conf __pycache__/ +build/ +dist/ +*.egg-info/ audit-reports/ coverage/ htmlcov/ diff --git a/README.md b/README.md index a4fa62a..f738a04 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,18 @@ Generate the CycloneDX dependency inventory from a resolved release environment: ./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python ``` +Synchronize module package workflows and inspect the registry release contract: + +```sh +./.venv/bin/python tools/repo/sync-module-package-workflows.py --check +./.venv/bin/python tools/release/generate-release-package-set.py \ + --output /tmp/govoplan-release-packages.json +``` + +Package publication, exact artifact locking, and the optional `govoplan` +developer meta-package are documented in +[Package Registry Releases](docs/PACKAGE_REGISTRY_RELEASES.md). + For reproducible release artifacts, set `SOURCE_DATE_EPOCH` to the release commit timestamp (or pass an explicit timezone-qualified `--timestamp`): diff --git a/docs/PACKAGE_REGISTRY_RELEASES.md b/docs/PACKAGE_REGISTRY_RELEASES.md new file mode 100644 index 0000000..4d5aca0 --- /dev/null +++ b/docs/PACKAGE_REGISTRY_RELEASES.md @@ -0,0 +1,109 @@ +# Package Registry Releases + +GovOPlaN publishes reusable module artifacts through Gitea's native PyPI and +npm registries. These packages improve developer installation, release +resolution, cacheability, and artifact inspection. They do not replace the +signed runtime distribution: the signed manifest and digest-pinned OCI images +remain the production deployment authority. + +## Publication boundary + +Every repository with a `pyproject.toml` contains +`.gitea/workflows/module-package-release.yml`. The meta repository owns the +canonical template and installs it with: + +```bash +python tools/repo/sync-module-package-workflows.py --write +python tools/repo/sync-module-package-workflows.py --check +``` + +The workflow runs for `v*` tags and may be dispatched manually for an existing +tag. Before building, it verifies that: + +- the selected tag is covered by repository tag protection; +- the tagged commit is contained in `main`; +- the tag, Python project version, and optional WebUI package version agree; +- package names remain in the `govoplan-*` and `@govoplan/*-webui` namespaces. + +It builds one wheel and, where applicable, one npm tarball. The workflow records +the source tag, source commit, filename, size, and SHA-256 in +`package-artifacts.json` before publishing. Gitea rejects a second upload of the +same package version, so correction requires a new version rather than artifact +replacement. + +Published WebUI packages contain registry-compatible dependencies only. The +workflow converts an internal dependency pinned to a protected `vX.Y.Z` Git tag +into the exact `X.Y.Z` registry version and rejects unresolved `file:` or Git +dependencies. Repository development metadata may therefore keep local or Git +references without leaking them into the published package contract. + +## One-time Gitea setup + +Protect `v*` tags in every package repository and the meta repository. Allow +only the `Owners` team to create or delete those tags. + +```bash +set -a +. ~/.config/gitea/gitea.env +set +a +python tools/gitea/gitea-configure-package-releases.py --apply +``` + +Create a dedicated personal access token with only `write:package` scope and +store these organization-level Actions secrets on `GovOPlaN`: + +- `GOVOPLAN_PACKAGE_USERNAME`: account owning the package token; +- `GOVOPLAN_PACKAGE_TOKEN`: dedicated package-write token. + +Do not use an administrator or general release token. Gitea 1.24 does not grant +package publication to the automatic Actions job token. Organization secrets +allow the same least-privilege credential to serve every module workflow. + +## Exact release consumption + +`tools/release/generate-release-package-set.py` translates the reviewed Git +source refs in `requirements-release.txt` into an exact registry package set. +It resolves each version tag to its commit and verifies the package metadata in +that tag. + +`tools/release/resolve-package-artifacts.py` then downloads exactly those wheel +and WebUI versions from Gitea. It reads the identity embedded in every wheel and +npm tarball, rejects missing, duplicate, unexpected, or oversized artifacts, +and writes `package-artifacts.lock.json` with SHA-256 values and npm integrity +values. Credentials are accepted only through environment variables and are +never written to the lock. Python resolution ignores ambient pip configuration +and extra indexes for GovOPlaN roots, preventing an internal package name from +being selected from an undeclared registry. + +The runtime distribution workflow uses the verified wheelhouse directly and +installs module WebUI tarballs only after matching them to the lock. It publishes +the package set, package lock, and hash-locked requirements as release assets. +The package-lock SHA-256 is part of the signed distribution manifest. Runtime +finalization also requires the lock's package versions and hashes to match the +wheel composition embedded in the images. OCI assembly remains network-free +after package and third-party dependency resolution. + +The source refs remain in the module catalog for source provenance and release +planning. Production installation consumes the signed runtime images rather +than invoking `pip`, `npm`, or Git on the target host. + +## Developer meta-package + +`packages/govoplan-meta` builds the optional `govoplan` package. Its default +dependencies mirror the reviewed runtime roots; `govoplan[full]` adds all +currently packageable workspace modules. Regenerate it after changing release +requirements or package versions: + +```bash +python tools/release/generate-developer-meta-package.py +python tools/release/generate-developer-meta-package.py --check +``` + +`push-release-tag.sh` performs this synchronization before release commits and +tags. The meta-package is for editable/developer setup and composition tests. It +does not enable modules, apply migrations, provision services, or establish +backup and recovery evidence. + +Generic Packages are intentionally not used. Add that transport only when a +consumer needs an artifact format unsupported by PyPI, npm, Gitea Releases, or +the OCI registry. diff --git a/docs/REPOSITORY_STRUCTURE.md b/docs/REPOSITORY_STRUCTURE.md index 0cc971c..251f090 100644 --- a/docs/REPOSITORY_STRUCTURE.md +++ b/docs/REPOSITORY_STRUCTURE.md @@ -76,6 +76,13 @@ one place. If a deployment profile later needs pinned SHAs for every repository, generate that lock as a release artifact instead of making day-to-day development depend on submodule updates. +Module release tags also publish wheels and WebUI tarballs to the organization +PyPI/npm registries. The meta release resolves exact versions into a hash-bound +package lock before producing the signed OCI runtime. See +`docs/PACKAGE_REGISTRY_RELEASES.md`. Git tags remain source provenance; package +registries are reusable artifact transport; the signed runtime manifest and +digest-pinned images remain production authority. + ## Docker Placement Whole-product Docker and production-like deployment composition belongs in diff --git a/docs/runtime-distribution-manifest.schema.json b/docs/runtime-distribution-manifest.schema.json index d6451c1..d2f1a0c 100644 --- a/docs/runtime-distribution-manifest.schema.json +++ b/docs/runtime-distribution-manifest.schema.json @@ -13,6 +13,7 @@ "expires_at", "revoked", "deployer", + "package_lock", "images", "dependencies", "composition", @@ -27,6 +28,7 @@ "expires_at": { "type": "string", "format": "date-time" }, "revoked": { "const": false }, "deployer": { "$ref": "#/$defs/artifact" }, + "package_lock": { "$ref": "#/$defs/artifact" }, "images": { "type": "object", "additionalProperties": false, diff --git a/packages/govoplan-meta/README.md b/packages/govoplan-meta/README.md new file mode 100644 index 0000000..be3e41a --- /dev/null +++ b/packages/govoplan-meta/README.md @@ -0,0 +1,11 @@ +# GovOPlaN developer meta-package + +`govoplan` is an optional convenience package for local development and +composition tests. The default dependency set matches the reviewed runtime +release roots; `govoplan[full]` adds every packageable module present in the +workspace at generation time. + +This package is not a production deployment artifact. Production installations +consume the signed runtime distribution manifest and digest-pinned OCI images. +The package does not enable modules, apply migrations, choose infrastructure, +or replace installation and recovery evidence. diff --git a/packages/govoplan-meta/pyproject.toml b/packages/govoplan-meta/pyproject.toml new file mode 100644 index 0000000..88b2a7d --- /dev/null +++ b/packages/govoplan-meta/pyproject.toml @@ -0,0 +1,90 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "govoplan" +version = "0.1.14" +description = "Developer convenience package for a versioned GovOPlaN composition" +readme = "README.md" +requires-python = ">=3.12" +license = { text = "AGPL-3.0-or-later" } +dependencies = [ + "govoplan-core[server]==0.1.14", + "govoplan-tenancy==0.1.8", + "govoplan-organizations==0.1.8", + "govoplan-identity==0.1.8", + "govoplan-idm==0.1.8", + "govoplan-access==0.1.8", + "govoplan-admin==0.1.8", + "govoplan-policy==0.1.8", + "govoplan-audit==0.1.8", + "govoplan-dashboard==0.1.8", + "govoplan-files==0.1.8", + "govoplan-mail==0.1.10", + "govoplan-campaign==0.1.11", + "govoplan-calendar==0.1.8", + "govoplan-docs==0.1.8", + "govoplan-ops==0.1.8", +] + +[project.optional-dependencies] +full = [ + "govoplan-addresses==0.1.9", + "govoplan-approvals==0.1.14", + "govoplan-assets==0.1.8", + "govoplan-booking==0.1.8", + "govoplan-cases==0.1.8", + "govoplan-certificates==0.1.8", + "govoplan-committee==0.1.8", + "govoplan-connectors==0.1.14", + "govoplan-consultation==0.1.8", + "govoplan-contracts==0.1.8", + "govoplan-dataflow==0.1.14", + "govoplan-datasources==0.1.14", + "govoplan-decisions==0.1.14", + "govoplan-dist-lists==0.1.14", + "govoplan-encryption==0.1.14", + "govoplan-evaluation==0.1.8", + "govoplan-facilities==0.1.8", + "govoplan-forms==0.1.14", + "govoplan-forms-runtime==0.1.14", + "govoplan-grants==0.1.8", + "govoplan-helpdesk==0.1.8", + "govoplan-identity-trust==0.1.14", + "govoplan-inspections==0.1.8", + "govoplan-learning==0.1.8", + "govoplan-mandates==0.1.14", + "govoplan-notifications==0.1.8", + "govoplan-parties==0.1.14", + "govoplan-permits==0.1.8", + "govoplan-poll==0.1.11", + "govoplan-portal==0.1.8", + "govoplan-postbox==0.1.2", + "govoplan-procurement==0.1.8", + "govoplan-projects==0.1.14", + "govoplan-records==0.1.8", + "govoplan-reporting==0.1.14", + "govoplan-resources==0.1.8", + "govoplan-rest==0.1.7", + "govoplan-risk-compliance==0.1.8", + "govoplan-scheduling==0.1.11", + "govoplan-search==0.1.14", + "govoplan-services==0.1.14", + "govoplan-soap==0.1.7", + "govoplan-templates==0.1.14", + "govoplan-tickets==0.1.8", + "govoplan-transparency==0.1.8", + "govoplan-views==0.1.0", + "govoplan-voting==0.1.14", + "govoplan-wiki==0.1.14", + "govoplan-workflow==0.1.14", + "govoplan-workflow-engine==0.1.14", +] + +[project.urls] +Repository = "https://git.add-ideas.de/GovOPlaN/govoplan" +Documentation = "https://govoplan.add-ideas.de" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/packages/govoplan-meta/src/govoplan_meta/__init__.py b/packages/govoplan-meta/src/govoplan_meta/__init__.py new file mode 100644 index 0000000..cc2de5e --- /dev/null +++ b/packages/govoplan-meta/src/govoplan_meta/__init__.py @@ -0,0 +1,12 @@ +"""Metadata helpers for the optional GovOPlaN developer composition.""" + +from importlib.metadata import PackageNotFoundError, version + + +try: + __version__ = version("govoplan") +except PackageNotFoundError: # pragma: no cover - source checkout only + __version__ = "0+unknown" + + +__all__ = ["__version__"] diff --git a/tests/test_module_package_workflows.py b/tests/test_module_package_workflows.py new file mode 100644 index 0000000..3f60d2e --- /dev/null +++ b/tests/test_module_package_workflows.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +META_ROOT = Path(__file__).resolve().parents[1] + + +class ModulePackageWorkflowTests(unittest.TestCase): + def test_template_enforces_tag_version_hash_and_registry_contract(self) -> None: + workflow = ( + META_ROOT / "tools/repo/templates/module-package-release.yml" + ).read_text(encoding="utf-8") + + self.assertIn("tag_protections", workflow) + self.assertIn("git merge-base --is-ancestor", workflow) + self.assertIn("does not match", workflow) + self.assertIn("package-artifacts.json", workflow) + self.assertIn("api/packages/GovOPlaN/pypi", workflow) + self.assertIn("api/packages/GovOPlaN/npm", workflow) + self.assertIn("GOVOPLAN_PACKAGE_TOKEN", workflow) + self.assertIn("must resolve to an exact registry version", workflow) + self.assertIn("git\\\\.add-ideas\\\\.de/GovOPlaN", workflow) + self.assertIn("release package identity does not match", workflow) + self.assertNotIn("Generic", workflow) + + @unittest.skipUnless(shutil.which("node"), "Node.js is required") + def test_webui_publication_normalizes_internal_git_dependencies(self) -> None: + workflow = ( + META_ROOT / "tools/repo/templates/module-package-release.yml" + ).read_text(encoding="utf-8") + marker = " node <<'NODE'\n" + script = workflow.split(marker, 1)[1].split("\n NODE", 1)[0] + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + package_dir = root / ".package-webui" + package_dir.mkdir() + package_path = package_dir / "package.json" + package_path.write_text( + json.dumps( + { + "name": "@govoplan/core-webui", + "version": "0.1.14", + "private": True, + "dependencies": { + "@govoplan/access-webui": ( + "git+ssh://git@git.add-ideas.de/GovOPlaN/" + "govoplan-access.git#v0.1.11" + ) + }, + } + ), + encoding="utf-8", + ) + + subprocess.run( + ["node"], + input=script, + cwd=root, + check=True, + text=True, + capture_output=True, + ) + + package = json.loads(package_path.read_text(encoding="utf-8")) + self.assertNotIn("private", package) + self.assertEqual( + "0.1.11", package["dependencies"]["@govoplan/access-webui"] + ) + + def test_sync_script_only_targets_packageable_govoplan_repositories(self) -> None: + namespace: dict[str, object] = { + "__file__": str(META_ROOT / "tools/repo/sync-module-package-workflows.py"), + "__name__": "test_sync_module_package_workflows", + } + script = (META_ROOT / "tools/repo/sync-module-package-workflows.py").read_text( + encoding="utf-8" + ) + exec(compile(script, str(namespace["__file__"]), "exec"), namespace) + + with tempfile.TemporaryDirectory() as temporary: + parent = Path(temporary) + package_repositories = namespace["package_repositories"] + # The production inventory is authoritative, so a temporary parent + # only exposes matching paths that are present in that inventory. + known = parent / "govoplan-core" + known.mkdir() + (known / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + self.assertEqual(package_repositories(parent), (known,)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_package_registry_release.py b/tests/test_package_registry_release.py new file mode 100644 index 0000000..2e80f10 --- /dev/null +++ b/tests/test_package_registry_release.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from io import BytesIO +import importlib.util +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import tomllib +import unittest +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +PACKAGE_SET = _load( + "generate_release_package_set", + ROOT / "tools/release/generate-release-package-set.py", +) +ARTIFACTS = _load( + "resolve_package_artifacts", + ROOT / "tools/release/resolve-package-artifacts.py", +) + + +class PackageRegistryReleaseTests(unittest.TestCase): + def test_current_release_sources_form_a_hash_bound_package_set(self) -> None: + core_version = tomllib.loads( + (ROOT.parent / "govoplan-core/pyproject.toml").read_text(encoding="utf-8") + )["project"]["version"] + payload = PACKAGE_SET.generate_package_set( + core_version=core_version, + requirements=ROOT / "requirements-release.txt", + workspace=ROOT.parent, + ) + + self.assertEqual("1", payload["schema_version"]) + self.assertEqual("govoplan-core", payload["python"][0]["name"]) + self.assertIn( + "@govoplan/core-webui", + {item["name"] for item in payload["webui"]}, + ) + unsigned = dict(payload) + digest = unsigned.pop("package_set_sha256") + self.assertEqual(ARTIFACTS._canonical_sha256(unsigned), digest) + + def test_wheel_and_webui_artifacts_are_verified_by_embedded_identity(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-package-artifacts-") as value: + root = Path(value) + wheels = root / "wheels" + webui = root / "webui" + wheels.mkdir() + webui.mkdir() + wheel = wheels / "govoplan_demo-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr( + "govoplan_demo-1.2.3.dist-info/METADATA", + "Metadata-Version: 2.1\nName: govoplan-demo\nVersion: 1.2.3\n", + ) + package_json = json.dumps( + {"name": "@govoplan/demo-webui", "version": "1.2.3"} + ).encode("utf-8") + npm = webui / "govoplan-demo-webui-1.2.3.tgz" + with tarfile.open(npm, "w:gz") as archive: + member = tarfile.TarInfo("package/package.json") + member.size = len(package_json) + archive.addfile(member, BytesIO(package_json)) + source = { + "version": "1.2.3", + "repository": "govoplan-demo", + "tag": "v1.2.3", + "commit": "1" * 40, + } + + python_rows = ARTIFACTS._verify_wheels( + ({"name": "govoplan-demo", "extras": ["server"], **source},), wheels + ) + webui_rows = ARTIFACTS._verify_webui( + ({"name": "@govoplan/demo-webui", **source},), webui + ) + + self.assertEqual("govoplan-demo", python_rows[0]["name"]) + self.assertEqual(["server"], python_rows[0]["extras"]) + self.assertEqual("@govoplan/demo-webui", webui_rows[0]["name"]) + self.assertTrue(str(webui_rows[0]["integrity"]).startswith("sha512-")) + + def test_package_set_rejects_argument_shaped_package_names(self) -> None: + payload = { + "schema_version": "1", + "release_version": "1.2.3", + "registries": { + "python": "https://packages.example.test/pypi/simple", + "npm": "https://packages.example.test/npm/", + }, + "python": [ + { + "name": "--index-url", + "version": "1.2.3", + "repository": "govoplan-demo", + "extras": [], + "tag": "v1.2.3", + "commit": "1" * 40, + } + ], + "webui": [ + { + "name": "@govoplan/demo-webui", + "version": "1.2.3", + "repository": "govoplan-demo", + "tag": "v1.2.3", + "commit": "1" * 40, + } + ], + } + payload["package_set_sha256"] = ARTIFACTS._canonical_sha256(payload) + with tempfile.TemporaryDirectory() as value: + path = Path(value) / "packages.json" + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex( + ARTIFACTS.PackageArtifactError, "invalid identity" + ): + ARTIFACTS._load_package_set(path) + + def test_runtime_workflow_consumes_registry_artifacts_and_publishes_lock(self) -> None: + workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text( + encoding="utf-8" + ) + + self.assertIn("resolve-package-artifacts.py", workflow) + self.assertIn("package-artifacts.lock.json", workflow) + self.assertIn( + "--package-lock runtime-output/package-artifacts.lock.json", + workflow, + ) + self.assertNotIn( + "pip wheel --no-deps --wheel-dir runtime-output/local-wheels", + workflow, + ) + + def test_developer_meta_package_matches_workspace_versions(self) -> None: + script = _load( + "generate_developer_meta_package", + ROOT / "tools/release/generate-developer-meta-package.py", + ) + expected = script.render( + workspace=ROOT.parent, + requirements=ROOT / "requirements-release.txt", + ) + actual = (ROOT / "packages/govoplan-meta/pyproject.toml").read_text( + encoding="utf-8" + ) + self.assertEqual(expected, actual) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_distribution_build.py b/tests/test_runtime_distribution_build.py index 31b30d5..3db50bf 100644 --- a/tests/test_runtime_distribution_build.py +++ b/tests/test_runtime_distribution_build.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import hashlib import importlib.util import json import os @@ -298,12 +299,37 @@ class RuntimeDistributionBuildTests(unittest.TestCase): (root / "web.json").write_text(json.dumps(web_metadata)) deployer = root / "govoplan-deploy.pyz" deployer.write_bytes(b"zipapp") + package_lock = root / "package-artifacts.lock.json" + package_lock_value = { + "schema_version": "1", + "release_version": "1.2.3", + "python": [ + { + "name": "govoplan-core", + "version": "1.2.3", + "sha256": "8" * 64, + } + ], + "webui": [], + } + package_lock_value["lock_sha256"] = hashlib.sha256( + json.dumps( + package_lock_value, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + package_lock.write_text( + json.dumps(package_lock_value) + "\n", + encoding="utf-8", + ) args = argparse.Namespace( composition=root / "composition.json", api_metadata=root / "api.json", web_metadata=root / "web.json", deployer=deployer, deployer_url="https://downloads.example/govoplan-deploy.pyz", + package_lock=package_lock, artifact_base_url="https://downloads.example/runtime/v1.2.3", source_commit="f" * 40, version="1.2.3", @@ -327,6 +353,10 @@ class RuntimeDistributionBuildTests(unittest.TestCase): ) self.assertTrue((root / "evidence/api-sbom.cdx.json").is_file()) self.assertTrue((root / "evidence/web-provenance.json").is_file()) + self.assertEqual( + hashlib.sha256(package_lock.read_bytes()).hexdigest(), + descriptor["package_lock"]["sha256"], + ) def test_rejects_incomplete_oci_index(self) -> None: with self.assertRaisesRegex(ValueError, "linux/amd64 and linux/arm64"): diff --git a/tools/checks/check-focused.sh b/tools/checks/check-focused.sh index a9e273d..26f4d8d 100644 --- a/tools/checks/check-focused.sh +++ b/tools/checks/check-focused.sh @@ -40,6 +40,9 @@ GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" --require-architecture cd "$META_ROOT" +"$PYTHON" tools/repo/sync-module-package-workflows.py --check +"$PYTHON" tools/release/generate-developer-meta-package.py --check +"$PYTHON" -m unittest tests.test_module_package_workflows tests.test_package_registry_release "$PYTHON" -m unittest tests.test_deployment_installer "$PYTHON" -m unittest tests.test_capability_fit_evidence "$PYTHON" -m unittest tests.test_configuration_package_artifacts diff --git a/tools/deployment/govoplan_deploy/distribution.py b/tools/deployment/govoplan_deploy/distribution.py index af6c258..d256ed9 100644 --- a/tools/deployment/govoplan_deploy/distribution.py +++ b/tools/deployment/govoplan_deploy/distribution.py @@ -167,6 +167,7 @@ def validate_manifest( "composition", "signatures", }, + optional={"package_lock"}, label="distribution manifest", ) if payload.get("schema_version") != "1": @@ -200,6 +201,12 @@ def validate_manifest( _https_url(deployer.get("url"), "deployer.url") _sha256(deployer.get("sha256"), "deployer.sha256") + if "package_lock" in payload: + package_lock = _object(payload.get("package_lock"), "package_lock") + _exact_keys(package_lock, required={"url", "sha256"}, label="package_lock") + _https_url(package_lock.get("url"), "package_lock.url") + _sha256(package_lock.get("sha256"), "package_lock.sha256") + images = _object(payload.get("images"), "images") if set(images) != {"api", "web"}: raise DistributionError("images must contain exactly api and web") @@ -630,10 +637,12 @@ def _exact_keys( value: Mapping[str, Any], *, required: set[str], + optional: set[str] | None = None, label: str, ) -> None: + optional = optional or set() missing = sorted(required - set(value)) - extra = sorted(set(value) - required) + extra = sorted(set(value) - required - optional) if missing or extra: detail = [] if missing: diff --git a/tools/gitea/gitea-configure-package-releases.py b/tools/gitea/gitea-configure-package-releases.py new file mode 100644 index 0000000..17f0140 --- /dev/null +++ b/tools/gitea/gitea-configure-package-releases.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Configure and verify protected GovOPlaN package-release boundaries.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +from gitea_common import ( + GiteaClient, + GiteaError, + RepoTarget, + load_dotenv, + org_path, + quote_path, + repo_path, + require_token, +) + + +META_ROOT = Path(__file__).resolve().parents[2] +REQUIRED_SECRETS = {"GOVOPLAN_PACKAGE_USERNAME", "GOVOPLAN_PACKAGE_TOKEN"} + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="https://git.add-ideas.de") + parser.add_argument("--owner", default="GovOPlaN") + parser.add_argument("--team", default="Owners") + parser.add_argument("--pattern", default="v*") + parser.add_argument("--env-file", type=Path) + parser.add_argument("--apply", action="store_true") + return parser + + +def package_repositories() -> tuple[str, ...]: + inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8")) + values = ["govoplan"] + for item in inventory["repositories"]: + name = str(item["name"]) + repository = META_ROOT.parent / str(item["path"]) + if name.startswith("govoplan-") and (repository / "pyproject.toml").is_file(): + values.append(name) + return tuple(sorted(set(values))) + + +def configure( + client: GiteaClient, + *, + owner: str, + team: str, + pattern: str, + apply: bool, +) -> tuple[str, ...]: + missing: list[str] = [] + expected = { + "name_pattern": pattern, + "whitelist_teams": [team], + "whitelist_usernames": [], + } + for repository in package_repositories(): + path = repo_path(owner, repository, "/tag_protections") + protections = client.request_json("GET", path) + matching = [ + item + for item in protections + if isinstance(item, dict) and item.get("name_pattern") == pattern + ] + if len(matching) == 1 and _matches(matching[0], expected): + print(f"protected {repository}:{pattern}") + continue + missing.append(repository) + if not apply: + print(f"would protect {repository}:{pattern}") + continue + if len(matching) == 1: + protection_id = matching[0].get("id") + client.request_json( + "PATCH", + f"{path}/{quote_path(str(protection_id))}", + body=expected, + ) + print(f"updated {repository}:{pattern}") + elif not matching: + client.request_json("POST", path, body=expected) + print(f"created {repository}:{pattern}") + else: + raise GiteaError(f"{repository} has duplicate {pattern!r} tag protections") + return tuple(missing) + + +def _matches(value: dict[str, object], expected: dict[str, object]) -> bool: + return ( + value.get("name_pattern") == expected["name_pattern"] + and sorted(value.get("whitelist_teams") or []) == expected["whitelist_teams"] + and sorted(value.get("whitelist_usernames") or []) == expected["whitelist_usernames"] + ) + + +def main() -> int: + args = build_parser().parse_args() + try: + load_dotenv(args.env_file) + token = require_token() + target = RepoTarget(base_url=args.url, owner=args.owner, repo="govoplan") + with GiteaClient(target, token) as client: + mismatches = configure( + client, + owner=args.owner, + team=args.team, + pattern=args.pattern, + apply=args.apply, + ) + secrets = client.request_json( + "GET", org_path(args.owner, "/actions/secrets"), query={"limit": 50} + ) + names = { + str(item.get("name") or "") + for item in secrets + if isinstance(item, dict) + } + missing_secrets = sorted(REQUIRED_SECRETS - names) + if missing_secrets: + print( + "Missing organization Actions secrets: " + ", ".join(missing_secrets), + file=sys.stderr, + ) + unresolved = (bool(mismatches) and not args.apply) or bool(missing_secrets) + if unresolved: + return 1 + print("Package release protection and credential names are configured.") + return 0 + except (GiteaError, OSError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/finalize-runtime-distribution.py b/tools/release/finalize-runtime-distribution.py index b2fbe42..21f3fad 100644 --- a/tools/release/finalize-runtime-distribution.py +++ b/tools/release/finalize-runtime-distribution.py @@ -26,6 +26,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--web-metadata", type=Path, required=True) parser.add_argument("--deployer", type=Path, required=True) parser.add_argument("--deployer-url", required=True) + parser.add_argument("--package-lock", type=Path, required=True) parser.add_argument("--artifact-base-url", required=True) parser.add_argument("--source-commit", required=True) parser.add_argument("--version", required=True) @@ -45,6 +46,11 @@ def build_parser() -> argparse.ArgumentParser: def finalize(args: argparse.Namespace) -> dict[str, Any]: composition = _json_object(args.composition) + _validate_package_lock( + _json_object(args.package_lock), + version=args.version, + composition=composition, + ) api = _image_metadata(_json_object(args.api_metadata), "api") web = _image_metadata(_json_object(args.web_metadata), "web") dependencies = dict(_dependency(value) for value in args.dependency) @@ -99,6 +105,10 @@ def finalize(args: argparse.Namespace) -> dict[str, Any]: "url": args.deployer_url, "sha256": _sha256_file(args.deployer), }, + "package_lock": { + "url": f"{artifact_base}/package-artifacts.lock.json", + "sha256": _sha256_file(args.package_lock), + }, "images": { "api": { **api, @@ -247,6 +257,57 @@ def _json_object(path: Path) -> dict[str, Any]: return value +def _validate_package_lock( + lock: dict[str, Any], + *, + version: str, + composition: dict[str, Any], +) -> None: + if lock.get("schema_version") != "1" or lock.get("release_version") != version: + raise ValueError("package lock schema or release version does not match") + unsigned = dict(lock) + expected_hash = unsigned.pop("lock_sha256", None) + actual_hash = hashlib.sha256( + json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + if expected_hash != actual_hash: + raise ValueError("package lock hash does not match its contents") + rows = lock.get("python") + if not isinstance(rows, list): + raise ValueError("package lock Python artifacts are missing") + locked = _package_identities(rows, name_key="name") + composed = _package_identities( + composition["python"]["packages"], + name_key="package", + ) + if locked != composed: + raise ValueError("package lock does not match the runtime wheel composition") + + +def _package_identities( + rows: list[object], + *, + name_key: str, +) -> dict[str, tuple[str, str]]: + identities: dict[str, tuple[str, str]] = {} + for row in rows: + if not isinstance(row, dict): + raise ValueError("package artifact identity is malformed") + name = row.get(name_key) + version = row.get("version") + digest = row.get("sha256") + if ( + not isinstance(name, str) + or not isinstance(version, str) + or not isinstance(digest, str) + or SHA256.fullmatch(digest) is None + or name in identities + ): + raise ValueError("package artifact identity is malformed or duplicated") + identities[name] = (version, digest) + return identities + + def _https_url(value: str, label: str) -> str: parsed = urlsplit(value) if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: diff --git a/tools/release/generate-developer-meta-package.py b/tools/release/generate-developer-meta-package.py new file mode 100644 index 0000000..99d60d8 --- /dev/null +++ b/tools/release/generate-developer-meta-package.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Generate the optional GovOPlaN developer convenience meta-package.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import tomllib + + +META_ROOT = Path(__file__).resolve().parents[2] +DIRECT = re.compile(r"^(govoplan-[a-z0-9-]+)(?:\[([^]]+)\])?\s+@\s+.*@v([A-Za-z0-9._+!-]+)$") +LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[([^]]+)\])?$") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", type=Path, default=META_ROOT.parent) + parser.add_argument( + "--requirements", + type=Path, + default=META_ROOT / "requirements-release.txt", + ) + parser.add_argument( + "--output", + type=Path, + default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml", + ) + parser.add_argument("--check", action="store_true") + return parser + + +def render(*, workspace: Path, requirements: Path) -> str: + core = tomllib.loads((workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8"))["project"] + version = str(core["version"]) + base: list[str] = [] + for raw in requirements.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + local = LOCAL_CORE.fullmatch(line) + if local: + extra = f"[{local.group(1)}]" if local.group(1) else "" + base.append(f"govoplan-core{extra}=={version}") + continue + match = DIRECT.fullmatch(line) + if match is None: + raise ValueError(f"unsupported release requirement: {line!r}") + extra = f"[{match.group(2)}]" if match.group(2) else "" + base.append(f"{match.group(1)}{extra}=={match.group(3)}") + + base_names = {_requirement_name(item) for item in base} + full: list[str] = [] + for project_path in sorted(workspace.glob("govoplan-*/pyproject.toml")): + project = tomllib.loads(project_path.read_text(encoding="utf-8"))["project"] + name = str(project.get("name") or "") + package_version = str(project.get("version") or "") + if name.startswith("govoplan-") and name not in base_names: + full.append(f"{name}=={package_version}") + + return "\n".join( + [ + "[build-system]", + 'requires = ["setuptools>=69", "wheel"]', + 'build-backend = "setuptools.build_meta"', + "", + "[project]", + 'name = "govoplan"', + f'version = {json.dumps(version)}', + 'description = "Developer convenience package for a versioned GovOPlaN composition"', + 'readme = "README.md"', + 'requires-python = ">=3.12"', + 'license = { text = "AGPL-3.0-or-later" }', + "dependencies = [", + *[f" {json.dumps(item)}," for item in base], + "]", + "", + "[project.optional-dependencies]", + "full = [", + *[f" {json.dumps(item)}," for item in full], + "]", + "", + "[project.urls]", + 'Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"', + 'Documentation = "https://govoplan.add-ideas.de"', + "", + "[tool.setuptools.packages.find]", + 'where = ["src"]', + "", + ] + ) + + +def _requirement_name(value: str) -> str: + return value.split("[", 1)[0].split("==", 1)[0] + + +def main() -> int: + args = build_parser().parse_args() + try: + expected = render( + workspace=args.workspace.expanduser().resolve(), + requirements=args.requirements.expanduser().resolve(), + ) + except (OSError, KeyError, ValueError, tomllib.TOMLDecodeError) as exc: + print(f"error: {exc}") + return 1 + output = args.output.expanduser() + current = output.read_text(encoding="utf-8") if output.is_file() else None + if args.check: + if current != expected: + print(f"error: developer meta-package is stale: {output}") + return 1 + print("Developer meta-package is synchronized.") + return 0 + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(expected, encoding="utf-8") + print(f"Developer meta-package written to {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/generate-release-package-set.py b/tools/release/generate-release-package-set.py new file mode 100644 index 0000000..3fd894d --- /dev/null +++ b/tools/release/generate-release-package-set.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Generate the exact registry package set for a GovOPlaN runtime release.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tomllib + + +META_ROOT = Path(__file__).resolve().parents[2] +NAME = re.compile(r"^govoplan-[a-z0-9-]+$") +VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$") +GIT_REQUIREMENT = re.compile( + r"^(?Pgovoplan-[a-z0-9-]+)(?:\[(?P[^]]+)\])?\s+@\s+" + r"(?Pgit\+[^\s]+/GovOPlaN/(?Pgovoplan-[a-z0-9-]+)\.git@v" + r"(?P[A-Za-z0-9._+!-]+))$" +) +LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P[^]]+)\])?$") + + +class PackageSetError(ValueError): + """Release source references cannot form an immutable package set.""" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--version", + help="Core/meta release version. Defaults to the workspace Core version.", + ) + parser.add_argument( + "--requirements", + type=Path, + default=META_ROOT / "requirements-release.txt", + ) + parser.add_argument("--workspace", type=Path, default=META_ROOT.parent) + parser.add_argument("--output", type=Path, required=True) + return parser + + +def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]: + values: list[dict[str, object]] = [] + for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + local = LOCAL_CORE.fullmatch(line) + if local: + values.append( + { + "name": "govoplan-core", + "version": core_version.removeprefix("v"), + "repository": "govoplan-core", + "extras": _extras(local.group("extras")), + } + ) + continue + match = GIT_REQUIREMENT.fullmatch(line) + if match is None: + raise PackageSetError( + f"unsupported release requirement at {path}:{line_number}: {line!r}" + ) + values.append( + { + "name": match.group("package"), + "version": match.group("version"), + "repository": match.group("repo"), + "extras": _extras(match.group("extras")), + } + ) + names = [str(item["name"]) for item in values] + if not values or names.count("govoplan-core") != 1 or len(names) != len(set(names)): + raise PackageSetError("release requirements must contain one Core and unique packages") + return tuple(values) + + +def generate_package_set( + *, + core_version: str, + requirements: Path, + workspace: Path, +) -> dict[str, object]: + core_version = core_version.removeprefix("v") + if VERSION.fullmatch(core_version) is None: + raise PackageSetError("release version is invalid") + python_packages: list[dict[str, object]] = [] + webui_packages: list[dict[str, object]] = [] + seen_webui: set[str] = set() + for requirement in parse_release_requirements(requirements, core_version=core_version): + repository = workspace / str(requirement["repository"]) + tag = f"v{requirement['version']}" + if not (repository / ".git").is_dir(): + raise PackageSetError(f"release repository is missing: {repository}") + commit = _git(repository, "rev-list", "-n", "1", tag) + if not commit: + raise PackageSetError(f"release tag is missing: {repository.name}@{tag}") + project = tomllib.loads(_git(repository, "show", f"{tag}:pyproject.toml"))["project"] + if project.get("name") != requirement["name"] or project.get("version") != requirement["version"]: + raise PackageSetError(f"tag metadata does not match {repository.name}@{tag}") + entry = { + **requirement, + "tag": tag, + "commit": commit, + } + python_packages.append(entry) + try: + webui_raw = _git(repository, "show", f"{tag}:webui/package.json") + except subprocess.CalledProcessError: + continue + webui = json.loads(webui_raw) + webui_name = webui.get("name") + if ( + not isinstance(webui_name, str) + or not webui_name.startswith("@govoplan/") + or webui.get("version") != requirement["version"] + or webui_name in seen_webui + ): + raise PackageSetError(f"WebUI tag metadata does not match {repository.name}@{tag}") + seen_webui.add(webui_name) + webui_packages.append( + { + "name": webui_name, + "version": requirement["version"], + "repository": requirement["repository"], + "tag": tag, + "commit": commit, + } + ) + payload: dict[str, object] = { + "schema_version": "1", + "release_version": core_version, + "registries": { + "python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple", + "npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/", + }, + "python": python_packages, + "webui": webui_packages, + } + payload["package_set_sha256"] = _canonical_sha256(payload) + return payload + + +def _extras(value: str | None) -> list[str]: + if not value: + return [] + extras = sorted({item.strip() for item in value.split(",") if item.strip()}) + if any(re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None for item in extras): + raise PackageSetError("release requirement contains an invalid extra") + return extras + + +def _git(repository: Path, *arguments: str) -> str: + return subprocess.check_output( + ["git", "-C", str(repository), *arguments], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + + +def _canonical_sha256(value: object) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def main() -> int: + args = build_parser().parse_args() + try: + workspace = args.workspace.expanduser().resolve() + version = args.version + if not version: + core = tomllib.loads( + (workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8") + ) + version = str(core["project"]["version"]) + payload = generate_package_set( + core_version=version, + requirements=args.requirements.expanduser().resolve(), + workspace=workspace, + ) + except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"error: {exc}") + return 1 + output = args.output.expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Release package set written to {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/install-webui-release-dependencies.sh b/tools/release/install-webui-release-dependencies.sh index e6df26a..3b7a20a 100644 --- a/tools/release/install-webui-release-dependencies.sh +++ b/tools/release/install-webui-release-dependencies.sh @@ -8,6 +8,9 @@ WEBUI_DIR="${1:-$CORE_ROOT/webui}" WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")" GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv" export GOVOPLAN_DEPS +PACKAGE_LOCK="${GOVOPLAN_WEBUI_PACKAGE_LOCK:-}" +PACKAGE_DIR="${GOVOPLAN_WEBUI_PACKAGE_DIR:-}" +PYTHON_BIN="${PYTHON:-python3}" trap 'rm -rf "$WORK_ROOT"' EXIT @@ -55,9 +58,69 @@ rm -f package-lock.json npm cache clean --force retry npm install --prefer-online +if [[ -n "$PACKAGE_LOCK" || -n "$PACKAGE_DIR" ]]; then + [[ -n "$PACKAGE_LOCK" && -n "$PACKAGE_DIR" ]] || { + echo "GOVOPLAN_WEBUI_PACKAGE_LOCK and GOVOPLAN_WEBUI_PACKAGE_DIR must be set together" >&2 + exit 1 + } + "$PYTHON_BIN" - "$PACKAGE_LOCK" "$PACKAGE_DIR" "$GOVOPLAN_DEPS" <<'PY' +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import sys + +lock_path = Path(sys.argv[1]).resolve() +package_dir = Path(sys.argv[2]).resolve() +output = Path(sys.argv[3]) +lock = json.loads(lock_path.read_text(encoding="utf-8")) +if lock.get("schema_version") != "1" or not isinstance(lock.get("webui"), list): + raise SystemExit("WebUI package lock is malformed") +unsigned = dict(lock) +expected_lock_hash = unsigned.pop("lock_sha256", None) +actual_lock_hash = hashlib.sha256( + json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8") +).hexdigest() +if expected_lock_hash != actual_lock_hash: + raise SystemExit("WebUI package lock hash does not match its contents") +rows = {} +for item in lock["webui"]: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise SystemExit("WebUI package lock contains a malformed artifact") + if item["name"] in rows: + raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}") + rows[item["name"]] = item +requested = [] +for line in output.read_text(encoding="utf-8").splitlines(): + if not line: + continue + name, _source_ref = line.split("\t", 1) + row = rows.get(name) + if not isinstance(row, dict): + raise SystemExit(f"WebUI package lock has no artifact for {name}") + filename = row.get("filename") + if not isinstance(filename, str) or Path(filename).name != filename: + raise SystemExit(f"WebUI package lock has an invalid filename for {name}") + artifact = package_dir / filename + if artifact.is_symlink() or not artifact.is_file(): + raise SystemExit(f"WebUI package artifact is missing for {name}") + encoded = artifact.read_bytes() + if len(encoded) != row.get("size") or hashlib.sha256(encoded).hexdigest() != row.get("sha256"): + raise SystemExit(f"WebUI package artifact hash does not match for {name}") + requested.append(f"{name}\tfile:{artifact}") +output.write_text("\n".join(requested) + "\n", encoding="utf-8") +PY +fi + module_paths=() while IFS=$'\t' read -r package_name spec; do [[ -n "${package_name:-}" ]] || continue + if [[ "$spec" == file:* ]]; then + echo "Installing $package_name from verified package artifact" + module_paths+=("$spec") + continue + fi git_url="${spec%%#*}" git_ref="${spec#*#}" if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then diff --git a/tools/release/push-release-tag.sh b/tools/release/push-release-tag.sh index 076a6d9..1d9ed63 100644 --- a/tools/release/push-release-tag.sh +++ b/tools/release/push-release-tag.sh @@ -456,6 +456,13 @@ path.write_text(updated) PYCODE } +update_developer_meta_package() { + "$PYTHON" "$META_ROOT/tools/release/generate-developer-meta-package.py" \ + --workspace "$PARENT" \ + --requirements "$META_ROOT/requirements-release.txt" \ + --output "$META_ROOT/packages/govoplan-meta/pyproject.toml" +} + update_version_files() { local repo="$1" local version="$2" @@ -875,8 +882,10 @@ done if [[ "$DRY_RUN" -eq 1 ]]; then echo "Would update $META_ROOT/requirements-release.txt to $TAG" + echo "Would synchronize packages/govoplan-meta/pyproject.toml" else update_release_requirements "$TARGET_VERSION" + update_developer_meta_package fi refresh_development_webui_lock diff --git a/tools/release/resolve-package-artifacts.py b/tools/release/resolve-package-artifacts.py new file mode 100644 index 0000000..6470e44 --- /dev/null +++ b/tools/release/resolve-package-artifacts.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Download, verify, and lock exact GovOPlaN registry package artifacts.""" + +from __future__ import annotations + +import argparse +import base64 +from email.parser import BytesParser +from email.policy import compat32 +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from urllib.parse import quote, urlsplit, urlunsplit +import zipfile + + +NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +WEBUI_NAME = re.compile(r"^@govoplan/[a-z0-9]+(?:-[a-z0-9]+)*-webui$") +VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") +MAX_ARTIFACT_BYTES = 512 * 1024 * 1024 + + +class PackageArtifactError(ValueError): + """Registry artifacts do not match the selected package set.""" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package-set", type=Path, required=True) + parser.add_argument("--wheelhouse", type=Path, required=True) + parser.add_argument("--webui-packages", type=Path, required=True) + parser.add_argument("--lock-output", type=Path, required=True) + parser.add_argument("--requirements-output", type=Path) + parser.add_argument("--python", default=sys.executable) + parser.add_argument("--npm", default="npm") + return parser + + +def resolve(args: argparse.Namespace) -> dict[str, object]: + package_set = _load_package_set(args.package_set) + wheelhouse = args.wheelhouse.expanduser().resolve() + webui_packages = args.webui_packages.expanduser().resolve() + _require_empty_destination(wheelhouse) + _require_empty_destination(webui_packages) + wheelhouse.parent.mkdir(parents=True, exist_ok=True) + webui_packages.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="govoplan-package-resolution-") as value: + temporary = Path(value) + wheels = temporary / "wheels" + webui = temporary / "webui" + wheels.mkdir() + webui.mkdir() + _download_wheels( + packages=tuple(package_set["python"]), + destination=wheels, + python=args.python, + index_url=str(package_set["registries"]["python"]), + ) + _download_webui( + packages=tuple(package_set["webui"]), + destination=webui, + npm=args.npm, + registry=str(package_set["registries"]["npm"]), + ) + python_rows = _verify_wheels(tuple(package_set["python"]), wheels) + webui_rows = _verify_webui(tuple(package_set["webui"]), webui) + lock: dict[str, object] = { + "schema_version": "1", + "release_version": package_set["release_version"], + "package_set_sha256": package_set["package_set_sha256"], + "registries": package_set["registries"], + "python": python_rows, + "webui": webui_rows, + } + lock["lock_sha256"] = _canonical_sha256(lock) + shutil.copytree(wheels, wheelhouse, dirs_exist_ok=True) + shutil.copytree(webui, webui_packages, dirs_exist_ok=True) + args.lock_output.parent.mkdir(parents=True, exist_ok=True) + args.lock_output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if args.requirements_output is not None: + _write_requirements(args.requirements_output, python_rows) + return lock + + +def _load_package_set(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or value.get("schema_version") != "1": + raise PackageArtifactError("package set has an unsupported shape") + expected_hash = value.get("package_set_sha256") + unsigned = dict(value) + unsigned.pop("package_set_sha256", None) + if expected_hash != _canonical_sha256(unsigned): + raise PackageArtifactError("package set hash does not match its contents") + registries = value.get("registries") + if not isinstance(registries, dict) or set(registries) != {"python", "npm"}: + raise PackageArtifactError("package set registries are invalid") + for registry in registries.values(): + parsed = urlsplit(str(registry)) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: + raise PackageArtifactError("package registries must use credential-free HTTPS URLs") + for group in ("python", "webui"): + packages = value.get(group) + if not isinstance(packages, list) or not packages: + raise PackageArtifactError(f"package set {group} entries are missing") + _validate_package_entries(group, packages) + return value + + +def _validate_package_entries(group: str, packages: list[object]) -> None: + names: set[str] = set() + for raw in packages: + if not isinstance(raw, dict): + raise PackageArtifactError(f"package set {group} entry is malformed") + name = raw.get("name") + version = raw.get("version") + repository = raw.get("repository") + tag = raw.get("tag") + commit = raw.get("commit") + name_valid = ( + isinstance(name, str) + and (NAME.fullmatch(name) if group == "python" else WEBUI_NAME.fullmatch(name)) + ) + if ( + not name_valid + or name in names + or not isinstance(version, str) + or VERSION.fullmatch(version) is None + or not isinstance(repository, str) + or NAME.fullmatch(repository) is None + or tag != f"v{version}" + or not isinstance(commit, str) + or COMMIT.fullmatch(commit) is None + ): + raise PackageArtifactError(f"package set {group} entry has an invalid identity") + names.add(name) + if group == "python": + extras = raw.get("extras") + if not isinstance(extras, list) or any( + not isinstance(item, str) + or re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None + for item in extras + ): + raise PackageArtifactError("package set Python extras are invalid") + + +def _download_wheels( + *, packages: tuple[dict[str, object], ...], destination: Path, python: str, index_url: str +) -> None: + requirements = [_python_requirement(item) for item in packages] + environment = dict(os.environ) + environment["PIP_INDEX_URL"] = _authenticated_url(index_url) + environment["PIP_EXTRA_INDEX_URL"] = "" + environment["PIP_CONFIG_FILE"] = os.devnull + environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + subprocess.run( + [python, "-m", "pip", "download", "--no-deps", "--only-binary=:all:", "--dest", str(destination), *requirements], + check=True, + env=environment, + ) + + +def _download_webui( + *, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str +) -> None: + environment = dict(os.environ) + npmrc: tempfile.NamedTemporaryFile[bytes] | None = None + token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "") + if token: + parsed = urlsplit(registry) + auth_path = f"//{parsed.netloc}{parsed.path}:_authToken={token}\n" + npmrc = tempfile.NamedTemporaryFile(prefix="govoplan-npmrc-", delete=False) + npmrc.write(f"@govoplan:registry={registry}\n{auth_path}".encode("utf-8")) + npmrc.close() + os.chmod(npmrc.name, 0o600) + environment["NPM_CONFIG_USERCONFIG"] = npmrc.name + try: + for item in packages: + subprocess.run( + [npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry], + check=True, + env=environment, + ) + finally: + if npmrc is not None: + Path(npmrc.name).unlink(missing_ok=True) + + +def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: + expected = {_normalize(str(item["name"])): item for item in packages} + rows: list[dict[str, object]] = [] + seen: set[str] = set() + for path in sorted(root.glob("*.whl")): + identity = _wheel_identity(path) + name = str(identity["name"]) + package = expected.get(name) + if package is None or identity["version"] != package["version"] or name in seen: + raise PackageArtifactError(f"unexpected wheel artifact: {path.name}") + seen.add(name) + rows.append(_artifact_row(path, package)) + if seen != set(expected): + raise PackageArtifactError("registry did not return every selected Python wheel") + return sorted(rows, key=lambda item: str(item["name"])) + + +def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]: + expected = {str(item["name"]): item for item in packages} + rows: list[dict[str, object]] = [] + seen: set[str] = set() + for path in sorted(root.glob("*.tgz")): + identity = _npm_identity(path) + name = str(identity["name"]) + package = expected.get(name) + if package is None or identity["version"] != package["version"] or name in seen: + raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}") + seen.add(name) + row = _artifact_row(path, package) + row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii") + rows.append(row) + if seen != set(expected): + raise PackageArtifactError("registry did not return every selected WebUI package") + return sorted(rows, key=lambda item: str(item["name"])) + + +def _wheel_identity(path: Path) -> dict[str, str]: + _bounded(path) + with zipfile.ZipFile(path) as archive: + metadata = [ + item for item in archive.infolist() + if PurePosixPath(item.filename).name == "METADATA" + and PurePosixPath(item.filename).parent.name.endswith(".dist-info") + ] + if len(metadata) != 1 or metadata[0].file_size > 1024 * 1024: + raise PackageArtifactError(f"wheel metadata is invalid: {path.name}") + parsed = BytesParser(policy=compat32).parsebytes(archive.read(metadata[0])) + name = _normalize(str(parsed.get("Name") or "")) + version = str(parsed.get("Version") or "") + if NAME.fullmatch(name) is None or VERSION.fullmatch(version) is None: + raise PackageArtifactError(f"wheel identity is invalid: {path.name}") + return {"name": name, "version": version} + + +def _npm_identity(path: Path) -> dict[str, str]: + _bounded(path) + with tarfile.open(path, mode="r:gz") as archive: + try: + member = archive.getmember("package/package.json") + except KeyError as exc: + raise PackageArtifactError(f"npm package metadata is missing: {path.name}") from exc + if not member.isfile() or member.size > 1024 * 1024: + raise PackageArtifactError(f"npm package metadata is invalid: {path.name}") + extracted = archive.extractfile(member) + if extracted is None: + raise PackageArtifactError(f"npm package metadata cannot be read: {path.name}") + value = json.load(extracted) + name = value.get("name") + version = value.get("version") + if not isinstance(name, str) or not name.startswith("@govoplan/") or not isinstance(version, str) or VERSION.fullmatch(version) is None: + raise PackageArtifactError(f"npm package identity is invalid: {path.name}") + return {"name": name, "version": version} + + +def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]: + row = { + "name": package["name"], + "version": package["version"], + "repository": package["repository"], + "tag": package["tag"], + "commit": package["commit"], + "filename": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "size": path.stat().st_size, + } + if "extras" in package: + row["extras"] = package["extras"] + return row + + +def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None: + lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"] + for row in rows: + selected_extras = row.get("extras") or [] + extras = ( + f"[{','.join(str(value) for value in selected_extras)}]" + if selected_extras + else "" + ) + lines.append( + f"{row['name']}{extras}=={row['version']} " + f"--hash=sha256:{row['sha256']}" + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _python_requirement(item: dict[str, object]) -> str: + extras = item.get("extras") or [] + suffix = f"[{','.join(str(value) for value in extras)}]" if extras else "" + return f"{item['name']}{suffix}=={item['version']}" + + +def _authenticated_url(url: str) -> str: + token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "") + username = os.environ.get("GOVOPLAN_PACKAGE_USERNAME", "") + if not token: + return url + if not username: + raise PackageArtifactError("GOVOPLAN_PACKAGE_USERNAME is required with a package token") + parsed = urlsplit(url) + return urlunsplit( + ( + parsed.scheme, + f"{quote(username, safe='')}:{quote(token, safe='')}@{parsed.netloc}", + parsed.path, + parsed.query, + "", + ) + ) + + +def _require_empty_destination(path: Path) -> None: + if path.exists() and (not path.is_dir() or any(path.iterdir())): + raise PackageArtifactError(f"output directory must be absent or empty: {path}") + if path.is_symlink(): + raise PackageArtifactError(f"output directory must not be a symlink: {path}") + + +def _bounded(path: Path) -> None: + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_ARTIFACT_BYTES: + raise PackageArtifactError(f"package artifact is invalid or too large: {path.name}") + + +def _normalize(value: str) -> str: + return re.sub(r"[-_.]+", "-", value.strip().lower()) + + +def _canonical_sha256(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + + +def main() -> int: + args = build_parser().parse_args() + try: + lock = resolve(args) + except (PackageArtifactError, OSError, ValueError, subprocess.CalledProcessError, zipfile.BadZipFile, tarfile.TarError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(f"Resolved {len(lock['python'])} Python and {len(lock['webui'])} WebUI packages.") + print(f"Package artifact lock written to {args.lock_output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/repo/sync-module-package-workflows.py b/tools/repo/sync-module-package-workflows.py new file mode 100644 index 0000000..cfa881c --- /dev/null +++ b/tools/repo/sync-module-package-workflows.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Install or verify the canonical package-release workflow in module repos.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + + +META_ROOT = Path(__file__).resolve().parents[2] +TEMPLATE = META_ROOT / "tools" / "repo" / "templates" / "module-package-release.yml" +DESTINATION = Path(".gitea/workflows/module-package-release.yml") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true") + mode.add_argument("--write", action="store_true") + parser.add_argument( + "--parent", + type=Path, + default=META_ROOT.parent, + help="Parent directory containing the repositories.", + ) + return parser + + +def package_repositories(parent: Path) -> tuple[Path, ...]: + inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8")) + repositories: list[Path] = [] + for item in inventory["repositories"]: + name = str(item["name"]) + if not name.startswith("govoplan-"): + continue + repository = parent / str(item["path"]) + if (repository / "pyproject.toml").is_file(): + repositories.append(repository) + return tuple(sorted(repositories)) + + +def synchronize(*, parent: Path, write: bool) -> tuple[str, ...]: + expected = TEMPLATE.read_bytes() + mismatches: list[str] = [] + for repository in package_repositories(parent): + destination = repository / DESTINATION + current = destination.read_bytes() if destination.is_file() else None + if current == expected: + continue + mismatches.append(repository.name) + if write: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(destination.suffix + ".tmp") + temporary.write_bytes(expected) + temporary.chmod(0o644) + temporary.replace(destination) + return tuple(mismatches) + + +def main() -> int: + args = build_parser().parse_args() + mismatches = synchronize(parent=args.parent.expanduser().resolve(), write=args.write) + if args.write: + print(f"Installed package-release workflow in {len(mismatches)} repositories.") + return 0 + if mismatches: + print("Package-release workflow is missing or stale in:", file=sys.stderr) + for repository in mismatches: + print(f"- {repository}", file=sys.stderr) + return 1 + print("Package-release workflows are synchronized.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/repo/templates/module-package-release.yml b/tools/repo/templates/module-package-release.yml new file mode 100644 index 0000000..3365ff2 --- /dev/null +++ b/tools/repo/templates/module-package-release.yml @@ -0,0 +1,209 @@ +name: Module Package Release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + release_tag: + description: Existing protected version tag to publish + required: true + type: string + +jobs: + publish-packages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: "22" + - name: Select and validate protected release tag + shell: bash + env: + REQUESTED_TAG: ${{ inputs.release_tag }} + TRIGGER_TAG: ${{ gitea.ref_name }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + set -euo pipefail + tag="${REQUESTED_TAG:-$TRIGGER_TAG}" + case "$tag" in + v[0-9]*.[0-9]*.[0-9]*) ;; + *) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;; + esac + git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main + tag_commit="$(git rev-list -n 1 "$tag")" + git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || { + echo "Release tag is not contained in main" >&2 + exit 1 + } + python - "$tag" <<'PY' + import fnmatch + import json + import os + import sys + import urllib.request + + tag = sys.argv[1] + repository = os.environ["GITEA_REPOSITORY"] + request = urllib.request.Request( + f"{os.environ['GITEA_API_URL']}/repos/{repository}/tag_protections", + headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + protections = json.load(response) + if not any(fnmatch.fnmatchcase(tag, item.get("name_pattern", "")) for item in protections): + raise SystemExit(f"Release tag {tag!r} is not covered by repository tag protection") + PY + git checkout --detach "$tag" + printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV" + printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV" + - name: Validate package versions + run: | + python - <<'PY' + import json + from pathlib import Path + import os + import re + import tomllib + + tag = os.environ["RELEASE_TAG"] + expected = tag.removeprefix("v") + project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"] + if project.get("version") != expected: + raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}") + if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None: + raise SystemExit("Python distribution name must use the govoplan-* namespace") + webui = Path("webui/package.json") + if webui.is_file(): + package = json.loads(webui.read_text(encoding="utf-8")) + if package.get("version") != expected: + raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}") + if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None: + raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace") + release = Path("webui/package.release.json") + if release.is_file(): + release_package = json.loads(release.read_text(encoding="utf-8")) + if ( + release_package.get("name") != package.get("name") + or release_package.get("version") != expected + ): + raise SystemExit("WebUI release package identity does not match package.json and the release tag") + PY + - name: Build immutable package artifacts + shell: bash + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0 + rm -rf dist .package-webui + python -m build --wheel --outdir dist + python -m twine check dist/*.whl + if [[ -f webui/package.json ]]; then + mkdir .package-webui + cp -a webui/. .package-webui/ + rm -rf .package-webui/node_modules .package-webui/dist + if [[ -f .package-webui/package.release.json ]]; then + cp .package-webui/package.release.json .package-webui/package.json + fi + node <<'NODE' + const fs = require("node:fs"); + const path = ".package-webui/package.json"; + const packageJson = JSON.parse(fs.readFileSync(path, "utf8")); + const groups = ["dependencies", "optionalDependencies", "peerDependencies"]; + for (const group of groups) { + for (const [name, specifier] of Object.entries(packageJson[group] || {})) { + if (!name.startsWith("@govoplan/")) continue; + if (typeof specifier !== "string") { + throw new Error(`${group}.${name} must use a string version`); + } + const packageSlug = name.slice("@govoplan/".length); + if (!packageSlug.endsWith("-webui")) { + throw new Error(`${group}.${name} is outside the WebUI package namespace`); + } + const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`; + const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const gitTag = specifier.match( + new RegExp( + `^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/GovOPlaN/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`, + ), + ); + if (gitTag) { + packageJson[group][name] = gitTag[1]; + continue; + } + if (specifier.startsWith("file:") || specifier.startsWith("git+")) { + throw new Error( + `${group}.${name} must resolve to an exact registry version for publication`, + ); + } + } + } + delete packageJson.private; + fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`); + NODE + npm pkg delete private --prefix .package-webui + (cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist) + fi + python - <<'PY' + import hashlib + import json + from pathlib import Path + import os + import subprocess + + artifacts = [] + for path in sorted(Path("dist").iterdir()): + if path.suffix not in {".whl", ".tgz"}: + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size}) + payload = { + "schema_version": "1", + "repository": os.environ["GITEA_REPOSITORY"], + "tag": os.environ["RELEASE_TAG"], + "commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "artifacts": artifacts, + } + Path("dist/package-artifacts.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + PY + - name: Retain package hash evidence + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 + with: + name: module-packages-${{ gitea.ref_name }} + path: dist/package-artifacts.json + - name: Publish wheel and WebUI package + shell: bash + env: + PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }} + PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }} + run: | + set -euo pipefail + test -n "$PACKAGE_USERNAME" + test -n "$PACKAGE_TOKEN" + TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \ + python -m twine upload --non-interactive \ + --repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \ + dist/*.whl + shopt -s nullglob + webui_packages=(dist/*.tgz) + if (( ${#webui_packages[@]} )); then + npmrc="$(mktemp)" + trap 'rm -f "$npmrc"' EXIT + chmod 600 "$npmrc" + printf '%s\n' \ + '@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \ + "//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \ + > "$npmrc" + NPM_CONFIG_USERCONFIG="$npmrc" npm publish "${webui_packages[0]}" \ + --ignore-scripts --access public \ + --registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/ + fi