Add registry-backed module package releases
This commit is contained in:
@@ -0,0 +1,60 @@
|
|||||||
|
name: Developer Meta-package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-package:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- name: Validate protected release tag and package version
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import fnmatch
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tomllib
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
tag = os.environ["GITEA_REF_NAME"]
|
||||||
|
project = tomllib.loads(Path("packages/govoplan-meta/pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if tag != f"v{project['version']}":
|
||||||
|
raise SystemExit("meta-package version does not match the release tag")
|
||||||
|
if subprocess.run(["git", "merge-base", "--is-ancestor", "HEAD", "origin/main"]).returncode:
|
||||||
|
raise SystemExit("release tag is not contained in main")
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"{os.environ['GITEA_API_URL']}/repos/{os.environ['GITEA_REPOSITORY']}/tag_protections",
|
||||||
|
headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request, timeout=30) as response:
|
||||||
|
protections = json.load(response)
|
||||||
|
if not any(fnmatch.fnmatchcase(tag, item.get("name_pattern", "")) for item in protections):
|
||||||
|
raise SystemExit("release tag is not protected")
|
||||||
|
PY
|
||||||
|
- name: Build and publish developer package
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
python -m build --wheel --outdir dist packages/govoplan-meta
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
@@ -25,6 +25,12 @@ jobs:
|
|||||||
- name: Bootstrap GovOPlaN repositories
|
- name: Bootstrap GovOPlaN repositories
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website
|
run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website
|
||||||
|
- name: Validate package publication contracts
|
||||||
|
working-directory: govoplan
|
||||||
|
run: |
|
||||||
|
python tools/repo/sync-module-package-workflows.py --check
|
||||||
|
python tools/release/generate-developer-meta-package.py --check
|
||||||
|
python -m unittest tests.test_module_package_workflows tests.test_package_registry_release
|
||||||
- name: Install backend release integration dependencies
|
- name: Install backend release integration dependencies
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -101,12 +101,27 @@ jobs:
|
|||||||
run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website
|
run: python tools/repo/bootstrap-repositories.py --parent .. --transport public-https --reuse-checkout-auth --exclude-repo addideas-govoplan-website
|
||||||
- name: Build release wheel roots and WebUI
|
- name: Build release wheel roots and WebUI
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
|
env:
|
||||||
|
VERSION: ${{ inputs.version }}
|
||||||
|
GOVOPLAN_PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
GOVOPLAN_PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
python -m venv .runtime-build
|
python -m venv .runtime-build
|
||||||
.runtime-build/bin/python -m pip install --upgrade pip wheel cryptography
|
.runtime-build/bin/python -m pip install --upgrade pip cryptography
|
||||||
mkdir -p runtime-output/local-wheels
|
.runtime-build/bin/python tools/release/generate-release-package-set.py \
|
||||||
.runtime-build/bin/python -m pip wheel --no-deps --wheel-dir runtime-output/local-wheels --requirement requirements-release.txt
|
--version "$VERSION" \
|
||||||
bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
|
--output runtime-output/release-packages.json
|
||||||
|
.runtime-build/bin/python tools/release/resolve-package-artifacts.py \
|
||||||
|
--package-set runtime-output/release-packages.json \
|
||||||
|
--wheelhouse runtime-output/local-wheels \
|
||||||
|
--webui-packages runtime-output/webui-packages \
|
||||||
|
--lock-output runtime-output/package-artifacts.lock.json \
|
||||||
|
--requirements-output runtime-output/requirements-release.packages.txt \
|
||||||
|
--python .runtime-build/bin/python
|
||||||
|
PYTHON=.runtime-build/bin/python \
|
||||||
|
GOVOPLAN_WEBUI_PACKAGE_LOCK="$PWD/runtime-output/package-artifacts.lock.json" \
|
||||||
|
GOVOPLAN_WEBUI_PACKAGE_DIR="$PWD/runtime-output/webui-packages" \
|
||||||
|
bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
|
||||||
npm --prefix ../govoplan-core/webui run build
|
npm --prefix ../govoplan-core/webui run build
|
||||||
.runtime-build/bin/python tools/release/prepare-runtime-context.py \
|
.runtime-build/bin/python tools/release/prepare-runtime-context.py \
|
||||||
--wheelhouse runtime-output/local-wheels \
|
--wheelhouse runtime-output/local-wheels \
|
||||||
@@ -264,6 +279,7 @@ jobs:
|
|||||||
--web-metadata runtime-output/web-metadata.json \
|
--web-metadata runtime-output/web-metadata.json \
|
||||||
--deployer runtime-output/govoplan-deploy.pyz \
|
--deployer runtime-output/govoplan-deploy.pyz \
|
||||||
--deployer-url "$ARTIFACT_BASE/govoplan-deploy.pyz" \
|
--deployer-url "$ARTIFACT_BASE/govoplan-deploy.pyz" \
|
||||||
|
--package-lock runtime-output/package-artifacts.lock.json \
|
||||||
--artifact-base-url "$ARTIFACT_BASE" \
|
--artifact-base-url "$ARTIFACT_BASE" \
|
||||||
--source-commit "$SOURCE_COMMIT" \
|
--source-commit "$SOURCE_COMMIT" \
|
||||||
--version "$VERSION" \
|
--version "$VERSION" \
|
||||||
@@ -361,6 +377,9 @@ jobs:
|
|||||||
--asset runtime-output/distribution-manifest.json.sha256 \
|
--asset runtime-output/distribution-manifest.json.sha256 \
|
||||||
--asset runtime-output/distribution-keyring.json \
|
--asset runtime-output/distribution-keyring.json \
|
||||||
--asset runtime-output/context-amd64/composition.json \
|
--asset runtime-output/context-amd64/composition.json \
|
||||||
|
--asset runtime-output/release-packages.json \
|
||||||
|
--asset runtime-output/package-artifacts.lock.json \
|
||||||
|
--asset runtime-output/requirements-release.packages.txt \
|
||||||
--asset runtime-output/evidence/api-sbom.cdx.json \
|
--asset runtime-output/evidence/api-sbom.cdx.json \
|
||||||
--asset runtime-output/evidence/web-sbom.cdx.json \
|
--asset runtime-output/evidence/web-sbom.cdx.json \
|
||||||
--asset runtime-output/evidence/api-provenance.json \
|
--asset runtime-output/evidence/api-provenance.json \
|
||||||
|
|||||||
@@ -10,6 +10,9 @@ tools/release/runtime/*
|
|||||||
!tools/release/runtime/Dockerfile.web
|
!tools/release/runtime/Dockerfile.web
|
||||||
!tools/release/runtime/nginx.conf
|
!tools/release/runtime/nginx.conf
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
*.egg-info/
|
||||||
audit-reports/
|
audit-reports/
|
||||||
coverage/
|
coverage/
|
||||||
htmlcov/
|
htmlcov/
|
||||||
|
|||||||
@@ -113,6 +113,18 @@ Generate the CycloneDX dependency inventory from a resolved release environment:
|
|||||||
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Synchronize module package workflows and inspect the registry release contract:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./.venv/bin/python tools/repo/sync-module-package-workflows.py --check
|
||||||
|
./.venv/bin/python tools/release/generate-release-package-set.py \
|
||||||
|
--output /tmp/govoplan-release-packages.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Package publication, exact artifact locking, and the optional `govoplan`
|
||||||
|
developer meta-package are documented in
|
||||||
|
[Package Registry Releases](docs/PACKAGE_REGISTRY_RELEASES.md).
|
||||||
|
|
||||||
For reproducible release artifacts, set `SOURCE_DATE_EPOCH` to the release
|
For reproducible release artifacts, set `SOURCE_DATE_EPOCH` to the release
|
||||||
commit timestamp (or pass an explicit timezone-qualified `--timestamp`):
|
commit timestamp (or pass an explicit timezone-qualified `--timestamp`):
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
# Package Registry Releases
|
||||||
|
|
||||||
|
GovOPlaN publishes reusable module artifacts through Gitea's native PyPI and
|
||||||
|
npm registries. These packages improve developer installation, release
|
||||||
|
resolution, cacheability, and artifact inspection. They do not replace the
|
||||||
|
signed runtime distribution: the signed manifest and digest-pinned OCI images
|
||||||
|
remain the production deployment authority.
|
||||||
|
|
||||||
|
## Publication boundary
|
||||||
|
|
||||||
|
Every repository with a `pyproject.toml` contains
|
||||||
|
`.gitea/workflows/module-package-release.yml`. The meta repository owns the
|
||||||
|
canonical template and installs it with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python tools/repo/sync-module-package-workflows.py --write
|
||||||
|
python tools/repo/sync-module-package-workflows.py --check
|
||||||
|
```
|
||||||
|
|
||||||
|
The workflow runs for `v*` tags and may be dispatched manually for an existing
|
||||||
|
tag. Before building, it verifies that:
|
||||||
|
|
||||||
|
- the selected tag is covered by repository tag protection;
|
||||||
|
- the tagged commit is contained in `main`;
|
||||||
|
- the tag, Python project version, and optional WebUI package version agree;
|
||||||
|
- package names remain in the `govoplan-*` and `@govoplan/*-webui` namespaces.
|
||||||
|
|
||||||
|
It builds one wheel and, where applicable, one npm tarball. The workflow records
|
||||||
|
the source tag, source commit, filename, size, and SHA-256 in
|
||||||
|
`package-artifacts.json` before publishing. Gitea rejects a second upload of the
|
||||||
|
same package version, so correction requires a new version rather than artifact
|
||||||
|
replacement.
|
||||||
|
|
||||||
|
Published WebUI packages contain registry-compatible dependencies only. The
|
||||||
|
workflow converts an internal dependency pinned to a protected `vX.Y.Z` Git tag
|
||||||
|
into the exact `X.Y.Z` registry version and rejects unresolved `file:` or Git
|
||||||
|
dependencies. Repository development metadata may therefore keep local or Git
|
||||||
|
references without leaking them into the published package contract.
|
||||||
|
|
||||||
|
## One-time Gitea setup
|
||||||
|
|
||||||
|
Protect `v*` tags in every package repository and the meta repository. Allow
|
||||||
|
only the `Owners` team to create or delete those tags.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
set -a
|
||||||
|
. ~/.config/gitea/gitea.env
|
||||||
|
set +a
|
||||||
|
python tools/gitea/gitea-configure-package-releases.py --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a dedicated personal access token with only `write:package` scope and
|
||||||
|
store these organization-level Actions secrets on `GovOPlaN`:
|
||||||
|
|
||||||
|
- `GOVOPLAN_PACKAGE_USERNAME`: account owning the package token;
|
||||||
|
- `GOVOPLAN_PACKAGE_TOKEN`: dedicated package-write token.
|
||||||
|
|
||||||
|
Do not use an administrator or general release token. Gitea 1.24 does not grant
|
||||||
|
package publication to the automatic Actions job token. Organization secrets
|
||||||
|
allow the same least-privilege credential to serve every module workflow.
|
||||||
|
|
||||||
|
## Exact release consumption
|
||||||
|
|
||||||
|
`tools/release/generate-release-package-set.py` translates the reviewed Git
|
||||||
|
source refs in `requirements-release.txt` into an exact registry package set.
|
||||||
|
It resolves each version tag to its commit and verifies the package metadata in
|
||||||
|
that tag.
|
||||||
|
|
||||||
|
`tools/release/resolve-package-artifacts.py` then downloads exactly those wheel
|
||||||
|
and WebUI versions from Gitea. It reads the identity embedded in every wheel and
|
||||||
|
npm tarball, rejects missing, duplicate, unexpected, or oversized artifacts,
|
||||||
|
and writes `package-artifacts.lock.json` with SHA-256 values and npm integrity
|
||||||
|
values. Credentials are accepted only through environment variables and are
|
||||||
|
never written to the lock. Python resolution ignores ambient pip configuration
|
||||||
|
and extra indexes for GovOPlaN roots, preventing an internal package name from
|
||||||
|
being selected from an undeclared registry.
|
||||||
|
|
||||||
|
The runtime distribution workflow uses the verified wheelhouse directly and
|
||||||
|
installs module WebUI tarballs only after matching them to the lock. It publishes
|
||||||
|
the package set, package lock, and hash-locked requirements as release assets.
|
||||||
|
The package-lock SHA-256 is part of the signed distribution manifest. Runtime
|
||||||
|
finalization also requires the lock's package versions and hashes to match the
|
||||||
|
wheel composition embedded in the images. OCI assembly remains network-free
|
||||||
|
after package and third-party dependency resolution.
|
||||||
|
|
||||||
|
The source refs remain in the module catalog for source provenance and release
|
||||||
|
planning. Production installation consumes the signed runtime images rather
|
||||||
|
than invoking `pip`, `npm`, or Git on the target host.
|
||||||
|
|
||||||
|
## Developer meta-package
|
||||||
|
|
||||||
|
`packages/govoplan-meta` builds the optional `govoplan` package. Its default
|
||||||
|
dependencies mirror the reviewed runtime roots; `govoplan[full]` adds all
|
||||||
|
currently packageable workspace modules. Regenerate it after changing release
|
||||||
|
requirements or package versions:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python tools/release/generate-developer-meta-package.py
|
||||||
|
python tools/release/generate-developer-meta-package.py --check
|
||||||
|
```
|
||||||
|
|
||||||
|
`push-release-tag.sh` performs this synchronization before release commits and
|
||||||
|
tags. The meta-package is for editable/developer setup and composition tests. It
|
||||||
|
does not enable modules, apply migrations, provision services, or establish
|
||||||
|
backup and recovery evidence.
|
||||||
|
|
||||||
|
Generic Packages are intentionally not used. Add that transport only when a
|
||||||
|
consumer needs an artifact format unsupported by PyPI, npm, Gitea Releases, or
|
||||||
|
the OCI registry.
|
||||||
@@ -76,6 +76,13 @@ one place. If a deployment profile later needs pinned SHAs for every repository,
|
|||||||
generate that lock as a release artifact instead of making day-to-day
|
generate that lock as a release artifact instead of making day-to-day
|
||||||
development depend on submodule updates.
|
development depend on submodule updates.
|
||||||
|
|
||||||
|
Module release tags also publish wheels and WebUI tarballs to the organization
|
||||||
|
PyPI/npm registries. The meta release resolves exact versions into a hash-bound
|
||||||
|
package lock before producing the signed OCI runtime. See
|
||||||
|
`docs/PACKAGE_REGISTRY_RELEASES.md`. Git tags remain source provenance; package
|
||||||
|
registries are reusable artifact transport; the signed runtime manifest and
|
||||||
|
digest-pinned images remain production authority.
|
||||||
|
|
||||||
## Docker Placement
|
## Docker Placement
|
||||||
|
|
||||||
Whole-product Docker and production-like deployment composition belongs in
|
Whole-product Docker and production-like deployment composition belongs in
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
"expires_at",
|
"expires_at",
|
||||||
"revoked",
|
"revoked",
|
||||||
"deployer",
|
"deployer",
|
||||||
|
"package_lock",
|
||||||
"images",
|
"images",
|
||||||
"dependencies",
|
"dependencies",
|
||||||
"composition",
|
"composition",
|
||||||
@@ -27,6 +28,7 @@
|
|||||||
"expires_at": { "type": "string", "format": "date-time" },
|
"expires_at": { "type": "string", "format": "date-time" },
|
||||||
"revoked": { "const": false },
|
"revoked": { "const": false },
|
||||||
"deployer": { "$ref": "#/$defs/artifact" },
|
"deployer": { "$ref": "#/$defs/artifact" },
|
||||||
|
"package_lock": { "$ref": "#/$defs/artifact" },
|
||||||
"images": {
|
"images": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# GovOPlaN developer meta-package
|
||||||
|
|
||||||
|
`govoplan` is an optional convenience package for local development and
|
||||||
|
composition tests. The default dependency set matches the reviewed runtime
|
||||||
|
release roots; `govoplan[full]` adds every packageable module present in the
|
||||||
|
workspace at generation time.
|
||||||
|
|
||||||
|
This package is not a production deployment artifact. Production installations
|
||||||
|
consume the signed runtime distribution manifest and digest-pinned OCI images.
|
||||||
|
The package does not enable modules, apply migrations, choose infrastructure,
|
||||||
|
or replace installation and recovery evidence.
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=69", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "govoplan"
|
||||||
|
version = "0.1.14"
|
||||||
|
description = "Developer convenience package for a versioned GovOPlaN composition"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
license = { text = "AGPL-3.0-or-later" }
|
||||||
|
dependencies = [
|
||||||
|
"govoplan-core[server]==0.1.14",
|
||||||
|
"govoplan-tenancy==0.1.8",
|
||||||
|
"govoplan-organizations==0.1.8",
|
||||||
|
"govoplan-identity==0.1.8",
|
||||||
|
"govoplan-idm==0.1.8",
|
||||||
|
"govoplan-access==0.1.8",
|
||||||
|
"govoplan-admin==0.1.8",
|
||||||
|
"govoplan-policy==0.1.8",
|
||||||
|
"govoplan-audit==0.1.8",
|
||||||
|
"govoplan-dashboard==0.1.8",
|
||||||
|
"govoplan-files==0.1.8",
|
||||||
|
"govoplan-mail==0.1.10",
|
||||||
|
"govoplan-campaign==0.1.11",
|
||||||
|
"govoplan-calendar==0.1.8",
|
||||||
|
"govoplan-docs==0.1.8",
|
||||||
|
"govoplan-ops==0.1.8",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
full = [
|
||||||
|
"govoplan-addresses==0.1.9",
|
||||||
|
"govoplan-approvals==0.1.14",
|
||||||
|
"govoplan-assets==0.1.8",
|
||||||
|
"govoplan-booking==0.1.8",
|
||||||
|
"govoplan-cases==0.1.8",
|
||||||
|
"govoplan-certificates==0.1.8",
|
||||||
|
"govoplan-committee==0.1.8",
|
||||||
|
"govoplan-connectors==0.1.14",
|
||||||
|
"govoplan-consultation==0.1.8",
|
||||||
|
"govoplan-contracts==0.1.8",
|
||||||
|
"govoplan-dataflow==0.1.14",
|
||||||
|
"govoplan-datasources==0.1.14",
|
||||||
|
"govoplan-decisions==0.1.14",
|
||||||
|
"govoplan-dist-lists==0.1.14",
|
||||||
|
"govoplan-encryption==0.1.14",
|
||||||
|
"govoplan-evaluation==0.1.8",
|
||||||
|
"govoplan-facilities==0.1.8",
|
||||||
|
"govoplan-forms==0.1.14",
|
||||||
|
"govoplan-forms-runtime==0.1.14",
|
||||||
|
"govoplan-grants==0.1.8",
|
||||||
|
"govoplan-helpdesk==0.1.8",
|
||||||
|
"govoplan-identity-trust==0.1.14",
|
||||||
|
"govoplan-inspections==0.1.8",
|
||||||
|
"govoplan-learning==0.1.8",
|
||||||
|
"govoplan-mandates==0.1.14",
|
||||||
|
"govoplan-notifications==0.1.8",
|
||||||
|
"govoplan-parties==0.1.14",
|
||||||
|
"govoplan-permits==0.1.8",
|
||||||
|
"govoplan-poll==0.1.11",
|
||||||
|
"govoplan-portal==0.1.8",
|
||||||
|
"govoplan-postbox==0.1.2",
|
||||||
|
"govoplan-procurement==0.1.8",
|
||||||
|
"govoplan-projects==0.1.14",
|
||||||
|
"govoplan-records==0.1.8",
|
||||||
|
"govoplan-reporting==0.1.14",
|
||||||
|
"govoplan-resources==0.1.8",
|
||||||
|
"govoplan-rest==0.1.7",
|
||||||
|
"govoplan-risk-compliance==0.1.8",
|
||||||
|
"govoplan-scheduling==0.1.11",
|
||||||
|
"govoplan-search==0.1.14",
|
||||||
|
"govoplan-services==0.1.14",
|
||||||
|
"govoplan-soap==0.1.7",
|
||||||
|
"govoplan-templates==0.1.14",
|
||||||
|
"govoplan-tickets==0.1.8",
|
||||||
|
"govoplan-transparency==0.1.8",
|
||||||
|
"govoplan-views==0.1.0",
|
||||||
|
"govoplan-voting==0.1.14",
|
||||||
|
"govoplan-wiki==0.1.14",
|
||||||
|
"govoplan-workflow==0.1.14",
|
||||||
|
"govoplan-workflow-engine==0.1.14",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"
|
||||||
|
Documentation = "https://govoplan.add-ideas.de"
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
"""Metadata helpers for the optional GovOPlaN developer composition."""
|
||||||
|
|
||||||
|
from importlib.metadata import PackageNotFoundError, version
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
__version__ = version("govoplan")
|
||||||
|
except PackageNotFoundError: # pragma: no cover - source checkout only
|
||||||
|
__version__ = "0+unknown"
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["__version__"]
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
class ModulePackageWorkflowTests(unittest.TestCase):
|
||||||
|
def test_template_enforces_tag_version_hash_and_registry_contract(self) -> None:
|
||||||
|
workflow = (
|
||||||
|
META_ROOT / "tools/repo/templates/module-package-release.yml"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
self.assertIn("tag_protections", workflow)
|
||||||
|
self.assertIn("git merge-base --is-ancestor", workflow)
|
||||||
|
self.assertIn("does not match", workflow)
|
||||||
|
self.assertIn("package-artifacts.json", workflow)
|
||||||
|
self.assertIn("api/packages/GovOPlaN/pypi", workflow)
|
||||||
|
self.assertIn("api/packages/GovOPlaN/npm", workflow)
|
||||||
|
self.assertIn("GOVOPLAN_PACKAGE_TOKEN", workflow)
|
||||||
|
self.assertIn("must resolve to an exact registry version", workflow)
|
||||||
|
self.assertIn("git\\\\.add-ideas\\\\.de/GovOPlaN", workflow)
|
||||||
|
self.assertIn("release package identity does not match", workflow)
|
||||||
|
self.assertNotIn("Generic", workflow)
|
||||||
|
|
||||||
|
@unittest.skipUnless(shutil.which("node"), "Node.js is required")
|
||||||
|
def test_webui_publication_normalizes_internal_git_dependencies(self) -> None:
|
||||||
|
workflow = (
|
||||||
|
META_ROOT / "tools/repo/templates/module-package-release.yml"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
marker = " node <<'NODE'\n"
|
||||||
|
script = workflow.split(marker, 1)[1].split("\n NODE", 1)[0]
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
package_dir = root / ".package-webui"
|
||||||
|
package_dir.mkdir()
|
||||||
|
package_path = package_dir / "package.json"
|
||||||
|
package_path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"name": "@govoplan/core-webui",
|
||||||
|
"version": "0.1.14",
|
||||||
|
"private": True,
|
||||||
|
"dependencies": {
|
||||||
|
"@govoplan/access-webui": (
|
||||||
|
"git+ssh://git@git.add-ideas.de/GovOPlaN/"
|
||||||
|
"govoplan-access.git#v0.1.11"
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.run(
|
||||||
|
["node"],
|
||||||
|
input=script,
|
||||||
|
cwd=root,
|
||||||
|
check=True,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
package = json.loads(package_path.read_text(encoding="utf-8"))
|
||||||
|
self.assertNotIn("private", package)
|
||||||
|
self.assertEqual(
|
||||||
|
"0.1.11", package["dependencies"]["@govoplan/access-webui"]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_sync_script_only_targets_packageable_govoplan_repositories(self) -> None:
|
||||||
|
namespace: dict[str, object] = {
|
||||||
|
"__file__": str(META_ROOT / "tools/repo/sync-module-package-workflows.py"),
|
||||||
|
"__name__": "test_sync_module_package_workflows",
|
||||||
|
}
|
||||||
|
script = (META_ROOT / "tools/repo/sync-module-package-workflows.py").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
exec(compile(script, str(namespace["__file__"]), "exec"), namespace)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
parent = Path(temporary)
|
||||||
|
package_repositories = namespace["package_repositories"]
|
||||||
|
# The production inventory is authoritative, so a temporary parent
|
||||||
|
# only exposes matching paths that are present in that inventory.
|
||||||
|
known = parent / "govoplan-core"
|
||||||
|
known.mkdir()
|
||||||
|
(known / "pyproject.toml").write_text("[project]\n", encoding="utf-8")
|
||||||
|
self.assertEqual(package_repositories(parent), (known,))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import tomllib
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _load(name: str, path: Path):
|
||||||
|
spec = importlib.util.spec_from_file_location(name, path)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
PACKAGE_SET = _load(
|
||||||
|
"generate_release_package_set",
|
||||||
|
ROOT / "tools/release/generate-release-package-set.py",
|
||||||
|
)
|
||||||
|
ARTIFACTS = _load(
|
||||||
|
"resolve_package_artifacts",
|
||||||
|
ROOT / "tools/release/resolve-package-artifacts.py",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PackageRegistryReleaseTests(unittest.TestCase):
|
||||||
|
def test_current_release_sources_form_a_hash_bound_package_set(self) -> None:
|
||||||
|
core_version = tomllib.loads(
|
||||||
|
(ROOT.parent / "govoplan-core/pyproject.toml").read_text(encoding="utf-8")
|
||||||
|
)["project"]["version"]
|
||||||
|
payload = PACKAGE_SET.generate_package_set(
|
||||||
|
core_version=core_version,
|
||||||
|
requirements=ROOT / "requirements-release.txt",
|
||||||
|
workspace=ROOT.parent,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("1", payload["schema_version"])
|
||||||
|
self.assertEqual("govoplan-core", payload["python"][0]["name"])
|
||||||
|
self.assertIn(
|
||||||
|
"@govoplan/core-webui",
|
||||||
|
{item["name"] for item in payload["webui"]},
|
||||||
|
)
|
||||||
|
unsigned = dict(payload)
|
||||||
|
digest = unsigned.pop("package_set_sha256")
|
||||||
|
self.assertEqual(ARTIFACTS._canonical_sha256(unsigned), digest)
|
||||||
|
|
||||||
|
def test_wheel_and_webui_artifacts_are_verified_by_embedded_identity(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-package-artifacts-") as value:
|
||||||
|
root = Path(value)
|
||||||
|
wheels = root / "wheels"
|
||||||
|
webui = root / "webui"
|
||||||
|
wheels.mkdir()
|
||||||
|
webui.mkdir()
|
||||||
|
wheel = wheels / "govoplan_demo-1.2.3-py3-none-any.whl"
|
||||||
|
with zipfile.ZipFile(wheel, "w") as archive:
|
||||||
|
archive.writestr(
|
||||||
|
"govoplan_demo-1.2.3.dist-info/METADATA",
|
||||||
|
"Metadata-Version: 2.1\nName: govoplan-demo\nVersion: 1.2.3\n",
|
||||||
|
)
|
||||||
|
package_json = json.dumps(
|
||||||
|
{"name": "@govoplan/demo-webui", "version": "1.2.3"}
|
||||||
|
).encode("utf-8")
|
||||||
|
npm = webui / "govoplan-demo-webui-1.2.3.tgz"
|
||||||
|
with tarfile.open(npm, "w:gz") as archive:
|
||||||
|
member = tarfile.TarInfo("package/package.json")
|
||||||
|
member.size = len(package_json)
|
||||||
|
archive.addfile(member, BytesIO(package_json))
|
||||||
|
source = {
|
||||||
|
"version": "1.2.3",
|
||||||
|
"repository": "govoplan-demo",
|
||||||
|
"tag": "v1.2.3",
|
||||||
|
"commit": "1" * 40,
|
||||||
|
}
|
||||||
|
|
||||||
|
python_rows = ARTIFACTS._verify_wheels(
|
||||||
|
({"name": "govoplan-demo", "extras": ["server"], **source},), wheels
|
||||||
|
)
|
||||||
|
webui_rows = ARTIFACTS._verify_webui(
|
||||||
|
({"name": "@govoplan/demo-webui", **source},), webui
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("govoplan-demo", python_rows[0]["name"])
|
||||||
|
self.assertEqual(["server"], python_rows[0]["extras"])
|
||||||
|
self.assertEqual("@govoplan/demo-webui", webui_rows[0]["name"])
|
||||||
|
self.assertTrue(str(webui_rows[0]["integrity"]).startswith("sha512-"))
|
||||||
|
|
||||||
|
def test_package_set_rejects_argument_shaped_package_names(self) -> None:
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"release_version": "1.2.3",
|
||||||
|
"registries": {
|
||||||
|
"python": "https://packages.example.test/pypi/simple",
|
||||||
|
"npm": "https://packages.example.test/npm/",
|
||||||
|
},
|
||||||
|
"python": [
|
||||||
|
{
|
||||||
|
"name": "--index-url",
|
||||||
|
"version": "1.2.3",
|
||||||
|
"repository": "govoplan-demo",
|
||||||
|
"extras": [],
|
||||||
|
"tag": "v1.2.3",
|
||||||
|
"commit": "1" * 40,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"webui": [
|
||||||
|
{
|
||||||
|
"name": "@govoplan/demo-webui",
|
||||||
|
"version": "1.2.3",
|
||||||
|
"repository": "govoplan-demo",
|
||||||
|
"tag": "v1.2.3",
|
||||||
|
"commit": "1" * 40,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
payload["package_set_sha256"] = ARTIFACTS._canonical_sha256(payload)
|
||||||
|
with tempfile.TemporaryDirectory() as value:
|
||||||
|
path = Path(value) / "packages.json"
|
||||||
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
ARTIFACTS.PackageArtifactError, "invalid identity"
|
||||||
|
):
|
||||||
|
ARTIFACTS._load_package_set(path)
|
||||||
|
|
||||||
|
def test_runtime_workflow_consumes_registry_artifacts_and_publishes_lock(self) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn("resolve-package-artifacts.py", workflow)
|
||||||
|
self.assertIn("package-artifacts.lock.json", workflow)
|
||||||
|
self.assertIn(
|
||||||
|
"--package-lock runtime-output/package-artifacts.lock.json",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
self.assertNotIn(
|
||||||
|
"pip wheel --no-deps --wheel-dir runtime-output/local-wheels",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_developer_meta_package_matches_workspace_versions(self) -> None:
|
||||||
|
script = _load(
|
||||||
|
"generate_developer_meta_package",
|
||||||
|
ROOT / "tools/release/generate-developer-meta-package.py",
|
||||||
|
)
|
||||||
|
expected = script.render(
|
||||||
|
workspace=ROOT.parent,
|
||||||
|
requirements=ROOT / "requirements-release.txt",
|
||||||
|
)
|
||||||
|
actual = (ROOT / "packages/govoplan-meta/pyproject.toml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -298,12 +299,37 @@ class RuntimeDistributionBuildTests(unittest.TestCase):
|
|||||||
(root / "web.json").write_text(json.dumps(web_metadata))
|
(root / "web.json").write_text(json.dumps(web_metadata))
|
||||||
deployer = root / "govoplan-deploy.pyz"
|
deployer = root / "govoplan-deploy.pyz"
|
||||||
deployer.write_bytes(b"zipapp")
|
deployer.write_bytes(b"zipapp")
|
||||||
|
package_lock = root / "package-artifacts.lock.json"
|
||||||
|
package_lock_value = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"release_version": "1.2.3",
|
||||||
|
"python": [
|
||||||
|
{
|
||||||
|
"name": "govoplan-core",
|
||||||
|
"version": "1.2.3",
|
||||||
|
"sha256": "8" * 64,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"webui": [],
|
||||||
|
}
|
||||||
|
package_lock_value["lock_sha256"] = hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
package_lock_value,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
package_lock.write_text(
|
||||||
|
json.dumps(package_lock_value) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
args = argparse.Namespace(
|
args = argparse.Namespace(
|
||||||
composition=root / "composition.json",
|
composition=root / "composition.json",
|
||||||
api_metadata=root / "api.json",
|
api_metadata=root / "api.json",
|
||||||
web_metadata=root / "web.json",
|
web_metadata=root / "web.json",
|
||||||
deployer=deployer,
|
deployer=deployer,
|
||||||
deployer_url="https://downloads.example/govoplan-deploy.pyz",
|
deployer_url="https://downloads.example/govoplan-deploy.pyz",
|
||||||
|
package_lock=package_lock,
|
||||||
artifact_base_url="https://downloads.example/runtime/v1.2.3",
|
artifact_base_url="https://downloads.example/runtime/v1.2.3",
|
||||||
source_commit="f" * 40,
|
source_commit="f" * 40,
|
||||||
version="1.2.3",
|
version="1.2.3",
|
||||||
@@ -327,6 +353,10 @@ class RuntimeDistributionBuildTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertTrue((root / "evidence/api-sbom.cdx.json").is_file())
|
self.assertTrue((root / "evidence/api-sbom.cdx.json").is_file())
|
||||||
self.assertTrue((root / "evidence/web-provenance.json").is_file())
|
self.assertTrue((root / "evidence/web-provenance.json").is_file())
|
||||||
|
self.assertEqual(
|
||||||
|
hashlib.sha256(package_lock.read_bytes()).hexdigest(),
|
||||||
|
descriptor["package_lock"]["sha256"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_rejects_incomplete_oci_index(self) -> None:
|
def test_rejects_incomplete_oci_index(self) -> None:
|
||||||
with self.assertRaisesRegex(ValueError, "linux/amd64 and linux/arm64"):
|
with self.assertRaisesRegex(ValueError, "linux/amd64 and linux/arm64"):
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash
|
|||||||
PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" --require-architecture
|
PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" --require-architecture
|
||||||
|
|
||||||
cd "$META_ROOT"
|
cd "$META_ROOT"
|
||||||
|
"$PYTHON" tools/repo/sync-module-package-workflows.py --check
|
||||||
|
"$PYTHON" tools/release/generate-developer-meta-package.py --check
|
||||||
|
"$PYTHON" -m unittest tests.test_module_package_workflows tests.test_package_registry_release
|
||||||
"$PYTHON" -m unittest tests.test_deployment_installer
|
"$PYTHON" -m unittest tests.test_deployment_installer
|
||||||
"$PYTHON" -m unittest tests.test_capability_fit_evidence
|
"$PYTHON" -m unittest tests.test_capability_fit_evidence
|
||||||
"$PYTHON" -m unittest tests.test_configuration_package_artifacts
|
"$PYTHON" -m unittest tests.test_configuration_package_artifacts
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ def validate_manifest(
|
|||||||
"composition",
|
"composition",
|
||||||
"signatures",
|
"signatures",
|
||||||
},
|
},
|
||||||
|
optional={"package_lock"},
|
||||||
label="distribution manifest",
|
label="distribution manifest",
|
||||||
)
|
)
|
||||||
if payload.get("schema_version") != "1":
|
if payload.get("schema_version") != "1":
|
||||||
@@ -200,6 +201,12 @@ def validate_manifest(
|
|||||||
_https_url(deployer.get("url"), "deployer.url")
|
_https_url(deployer.get("url"), "deployer.url")
|
||||||
_sha256(deployer.get("sha256"), "deployer.sha256")
|
_sha256(deployer.get("sha256"), "deployer.sha256")
|
||||||
|
|
||||||
|
if "package_lock" in payload:
|
||||||
|
package_lock = _object(payload.get("package_lock"), "package_lock")
|
||||||
|
_exact_keys(package_lock, required={"url", "sha256"}, label="package_lock")
|
||||||
|
_https_url(package_lock.get("url"), "package_lock.url")
|
||||||
|
_sha256(package_lock.get("sha256"), "package_lock.sha256")
|
||||||
|
|
||||||
images = _object(payload.get("images"), "images")
|
images = _object(payload.get("images"), "images")
|
||||||
if set(images) != {"api", "web"}:
|
if set(images) != {"api", "web"}:
|
||||||
raise DistributionError("images must contain exactly api and web")
|
raise DistributionError("images must contain exactly api and web")
|
||||||
@@ -630,10 +637,12 @@ def _exact_keys(
|
|||||||
value: Mapping[str, Any],
|
value: Mapping[str, Any],
|
||||||
*,
|
*,
|
||||||
required: set[str],
|
required: set[str],
|
||||||
|
optional: set[str] | None = None,
|
||||||
label: str,
|
label: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
optional = optional or set()
|
||||||
missing = sorted(required - set(value))
|
missing = sorted(required - set(value))
|
||||||
extra = sorted(set(value) - required)
|
extra = sorted(set(value) - required - optional)
|
||||||
if missing or extra:
|
if missing or extra:
|
||||||
detail = []
|
detail = []
|
||||||
if missing:
|
if missing:
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Configure and verify protected GovOPlaN package-release boundaries."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from gitea_common import (
|
||||||
|
GiteaClient,
|
||||||
|
GiteaError,
|
||||||
|
RepoTarget,
|
||||||
|
load_dotenv,
|
||||||
|
org_path,
|
||||||
|
quote_path,
|
||||||
|
repo_path,
|
||||||
|
require_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
REQUIRED_SECRETS = {"GOVOPLAN_PACKAGE_USERNAME", "GOVOPLAN_PACKAGE_TOKEN"}
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--url", default="https://git.add-ideas.de")
|
||||||
|
parser.add_argument("--owner", default="GovOPlaN")
|
||||||
|
parser.add_argument("--team", default="Owners")
|
||||||
|
parser.add_argument("--pattern", default="v*")
|
||||||
|
parser.add_argument("--env-file", type=Path)
|
||||||
|
parser.add_argument("--apply", action="store_true")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def package_repositories() -> tuple[str, ...]:
|
||||||
|
inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
|
||||||
|
values = ["govoplan"]
|
||||||
|
for item in inventory["repositories"]:
|
||||||
|
name = str(item["name"])
|
||||||
|
repository = META_ROOT.parent / str(item["path"])
|
||||||
|
if name.startswith("govoplan-") and (repository / "pyproject.toml").is_file():
|
||||||
|
values.append(name)
|
||||||
|
return tuple(sorted(set(values)))
|
||||||
|
|
||||||
|
|
||||||
|
def configure(
|
||||||
|
client: GiteaClient,
|
||||||
|
*,
|
||||||
|
owner: str,
|
||||||
|
team: str,
|
||||||
|
pattern: str,
|
||||||
|
apply: bool,
|
||||||
|
) -> tuple[str, ...]:
|
||||||
|
missing: list[str] = []
|
||||||
|
expected = {
|
||||||
|
"name_pattern": pattern,
|
||||||
|
"whitelist_teams": [team],
|
||||||
|
"whitelist_usernames": [],
|
||||||
|
}
|
||||||
|
for repository in package_repositories():
|
||||||
|
path = repo_path(owner, repository, "/tag_protections")
|
||||||
|
protections = client.request_json("GET", path)
|
||||||
|
matching = [
|
||||||
|
item
|
||||||
|
for item in protections
|
||||||
|
if isinstance(item, dict) and item.get("name_pattern") == pattern
|
||||||
|
]
|
||||||
|
if len(matching) == 1 and _matches(matching[0], expected):
|
||||||
|
print(f"protected {repository}:{pattern}")
|
||||||
|
continue
|
||||||
|
missing.append(repository)
|
||||||
|
if not apply:
|
||||||
|
print(f"would protect {repository}:{pattern}")
|
||||||
|
continue
|
||||||
|
if len(matching) == 1:
|
||||||
|
protection_id = matching[0].get("id")
|
||||||
|
client.request_json(
|
||||||
|
"PATCH",
|
||||||
|
f"{path}/{quote_path(str(protection_id))}",
|
||||||
|
body=expected,
|
||||||
|
)
|
||||||
|
print(f"updated {repository}:{pattern}")
|
||||||
|
elif not matching:
|
||||||
|
client.request_json("POST", path, body=expected)
|
||||||
|
print(f"created {repository}:{pattern}")
|
||||||
|
else:
|
||||||
|
raise GiteaError(f"{repository} has duplicate {pattern!r} tag protections")
|
||||||
|
return tuple(missing)
|
||||||
|
|
||||||
|
|
||||||
|
def _matches(value: dict[str, object], expected: dict[str, object]) -> bool:
|
||||||
|
return (
|
||||||
|
value.get("name_pattern") == expected["name_pattern"]
|
||||||
|
and sorted(value.get("whitelist_teams") or []) == expected["whitelist_teams"]
|
||||||
|
and sorted(value.get("whitelist_usernames") or []) == expected["whitelist_usernames"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
try:
|
||||||
|
load_dotenv(args.env_file)
|
||||||
|
token = require_token()
|
||||||
|
target = RepoTarget(base_url=args.url, owner=args.owner, repo="govoplan")
|
||||||
|
with GiteaClient(target, token) as client:
|
||||||
|
mismatches = configure(
|
||||||
|
client,
|
||||||
|
owner=args.owner,
|
||||||
|
team=args.team,
|
||||||
|
pattern=args.pattern,
|
||||||
|
apply=args.apply,
|
||||||
|
)
|
||||||
|
secrets = client.request_json(
|
||||||
|
"GET", org_path(args.owner, "/actions/secrets"), query={"limit": 50}
|
||||||
|
)
|
||||||
|
names = {
|
||||||
|
str(item.get("name") or "")
|
||||||
|
for item in secrets
|
||||||
|
if isinstance(item, dict)
|
||||||
|
}
|
||||||
|
missing_secrets = sorted(REQUIRED_SECRETS - names)
|
||||||
|
if missing_secrets:
|
||||||
|
print(
|
||||||
|
"Missing organization Actions secrets: " + ", ".join(missing_secrets),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
unresolved = (bool(mismatches) and not args.apply) or bool(missing_secrets)
|
||||||
|
if unresolved:
|
||||||
|
return 1
|
||||||
|
print("Package release protection and credential names are configured.")
|
||||||
|
return 0
|
||||||
|
except (GiteaError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -26,6 +26,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
parser.add_argument("--web-metadata", type=Path, required=True)
|
parser.add_argument("--web-metadata", type=Path, required=True)
|
||||||
parser.add_argument("--deployer", type=Path, required=True)
|
parser.add_argument("--deployer", type=Path, required=True)
|
||||||
parser.add_argument("--deployer-url", required=True)
|
parser.add_argument("--deployer-url", required=True)
|
||||||
|
parser.add_argument("--package-lock", type=Path, required=True)
|
||||||
parser.add_argument("--artifact-base-url", required=True)
|
parser.add_argument("--artifact-base-url", required=True)
|
||||||
parser.add_argument("--source-commit", required=True)
|
parser.add_argument("--source-commit", required=True)
|
||||||
parser.add_argument("--version", required=True)
|
parser.add_argument("--version", required=True)
|
||||||
@@ -45,6 +46,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
|
|
||||||
def finalize(args: argparse.Namespace) -> dict[str, Any]:
|
def finalize(args: argparse.Namespace) -> dict[str, Any]:
|
||||||
composition = _json_object(args.composition)
|
composition = _json_object(args.composition)
|
||||||
|
_validate_package_lock(
|
||||||
|
_json_object(args.package_lock),
|
||||||
|
version=args.version,
|
||||||
|
composition=composition,
|
||||||
|
)
|
||||||
api = _image_metadata(_json_object(args.api_metadata), "api")
|
api = _image_metadata(_json_object(args.api_metadata), "api")
|
||||||
web = _image_metadata(_json_object(args.web_metadata), "web")
|
web = _image_metadata(_json_object(args.web_metadata), "web")
|
||||||
dependencies = dict(_dependency(value) for value in args.dependency)
|
dependencies = dict(_dependency(value) for value in args.dependency)
|
||||||
@@ -99,6 +105,10 @@ def finalize(args: argparse.Namespace) -> dict[str, Any]:
|
|||||||
"url": args.deployer_url,
|
"url": args.deployer_url,
|
||||||
"sha256": _sha256_file(args.deployer),
|
"sha256": _sha256_file(args.deployer),
|
||||||
},
|
},
|
||||||
|
"package_lock": {
|
||||||
|
"url": f"{artifact_base}/package-artifacts.lock.json",
|
||||||
|
"sha256": _sha256_file(args.package_lock),
|
||||||
|
},
|
||||||
"images": {
|
"images": {
|
||||||
"api": {
|
"api": {
|
||||||
**api,
|
**api,
|
||||||
@@ -247,6 +257,57 @@ def _json_object(path: Path) -> dict[str, Any]:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_package_lock(
|
||||||
|
lock: dict[str, Any],
|
||||||
|
*,
|
||||||
|
version: str,
|
||||||
|
composition: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
if lock.get("schema_version") != "1" or lock.get("release_version") != version:
|
||||||
|
raise ValueError("package lock schema or release version does not match")
|
||||||
|
unsigned = dict(lock)
|
||||||
|
expected_hash = unsigned.pop("lock_sha256", None)
|
||||||
|
actual_hash = hashlib.sha256(
|
||||||
|
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
if expected_hash != actual_hash:
|
||||||
|
raise ValueError("package lock hash does not match its contents")
|
||||||
|
rows = lock.get("python")
|
||||||
|
if not isinstance(rows, list):
|
||||||
|
raise ValueError("package lock Python artifacts are missing")
|
||||||
|
locked = _package_identities(rows, name_key="name")
|
||||||
|
composed = _package_identities(
|
||||||
|
composition["python"]["packages"],
|
||||||
|
name_key="package",
|
||||||
|
)
|
||||||
|
if locked != composed:
|
||||||
|
raise ValueError("package lock does not match the runtime wheel composition")
|
||||||
|
|
||||||
|
|
||||||
|
def _package_identities(
|
||||||
|
rows: list[object],
|
||||||
|
*,
|
||||||
|
name_key: str,
|
||||||
|
) -> dict[str, tuple[str, str]]:
|
||||||
|
identities: dict[str, tuple[str, str]] = {}
|
||||||
|
for row in rows:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
raise ValueError("package artifact identity is malformed")
|
||||||
|
name = row.get(name_key)
|
||||||
|
version = row.get("version")
|
||||||
|
digest = row.get("sha256")
|
||||||
|
if (
|
||||||
|
not isinstance(name, str)
|
||||||
|
or not isinstance(version, str)
|
||||||
|
or not isinstance(digest, str)
|
||||||
|
or SHA256.fullmatch(digest) is None
|
||||||
|
or name in identities
|
||||||
|
):
|
||||||
|
raise ValueError("package artifact identity is malformed or duplicated")
|
||||||
|
identities[name] = (version, digest)
|
||||||
|
return identities
|
||||||
|
|
||||||
|
|
||||||
def _https_url(value: str, label: str) -> str:
|
def _https_url(value: str, label: str) -> str:
|
||||||
parsed = urlsplit(value)
|
parsed = urlsplit(value)
|
||||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate the optional GovOPlaN developer convenience meta-package."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
DIRECT = re.compile(r"^(govoplan-[a-z0-9-]+)(?:\[([^]]+)\])?\s+@\s+.*@v([A-Za-z0-9._+!-]+)$")
|
||||||
|
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[([^]]+)\])?$")
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||||
|
parser.add_argument(
|
||||||
|
"--requirements",
|
||||||
|
type=Path,
|
||||||
|
default=META_ROOT / "requirements-release.txt",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
|
||||||
|
)
|
||||||
|
parser.add_argument("--check", action="store_true")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def render(*, workspace: Path, requirements: Path) -> str:
|
||||||
|
core = tomllib.loads((workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
version = str(core["version"])
|
||||||
|
base: list[str] = []
|
||||||
|
for raw in requirements.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = raw.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
local = LOCAL_CORE.fullmatch(line)
|
||||||
|
if local:
|
||||||
|
extra = f"[{local.group(1)}]" if local.group(1) else ""
|
||||||
|
base.append(f"govoplan-core{extra}=={version}")
|
||||||
|
continue
|
||||||
|
match = DIRECT.fullmatch(line)
|
||||||
|
if match is None:
|
||||||
|
raise ValueError(f"unsupported release requirement: {line!r}")
|
||||||
|
extra = f"[{match.group(2)}]" if match.group(2) else ""
|
||||||
|
base.append(f"{match.group(1)}{extra}=={match.group(3)}")
|
||||||
|
|
||||||
|
base_names = {_requirement_name(item) for item in base}
|
||||||
|
full: list[str] = []
|
||||||
|
for project_path in sorted(workspace.glob("govoplan-*/pyproject.toml")):
|
||||||
|
project = tomllib.loads(project_path.read_text(encoding="utf-8"))["project"]
|
||||||
|
name = str(project.get("name") or "")
|
||||||
|
package_version = str(project.get("version") or "")
|
||||||
|
if name.startswith("govoplan-") and name not in base_names:
|
||||||
|
full.append(f"{name}=={package_version}")
|
||||||
|
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
"[build-system]",
|
||||||
|
'requires = ["setuptools>=69", "wheel"]',
|
||||||
|
'build-backend = "setuptools.build_meta"',
|
||||||
|
"",
|
||||||
|
"[project]",
|
||||||
|
'name = "govoplan"',
|
||||||
|
f'version = {json.dumps(version)}',
|
||||||
|
'description = "Developer convenience package for a versioned GovOPlaN composition"',
|
||||||
|
'readme = "README.md"',
|
||||||
|
'requires-python = ">=3.12"',
|
||||||
|
'license = { text = "AGPL-3.0-or-later" }',
|
||||||
|
"dependencies = [",
|
||||||
|
*[f" {json.dumps(item)}," for item in base],
|
||||||
|
"]",
|
||||||
|
"",
|
||||||
|
"[project.optional-dependencies]",
|
||||||
|
"full = [",
|
||||||
|
*[f" {json.dumps(item)}," for item in full],
|
||||||
|
"]",
|
||||||
|
"",
|
||||||
|
"[project.urls]",
|
||||||
|
'Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"',
|
||||||
|
'Documentation = "https://govoplan.add-ideas.de"',
|
||||||
|
"",
|
||||||
|
"[tool.setuptools.packages.find]",
|
||||||
|
'where = ["src"]',
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _requirement_name(value: str) -> str:
|
||||||
|
return value.split("[", 1)[0].split("==", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
try:
|
||||||
|
expected = render(
|
||||||
|
workspace=args.workspace.expanduser().resolve(),
|
||||||
|
requirements=args.requirements.expanduser().resolve(),
|
||||||
|
)
|
||||||
|
except (OSError, KeyError, ValueError, tomllib.TOMLDecodeError) as exc:
|
||||||
|
print(f"error: {exc}")
|
||||||
|
return 1
|
||||||
|
output = args.output.expanduser()
|
||||||
|
current = output.read_text(encoding="utf-8") if output.is_file() else None
|
||||||
|
if args.check:
|
||||||
|
if current != expected:
|
||||||
|
print(f"error: developer meta-package is stale: {output}")
|
||||||
|
return 1
|
||||||
|
print("Developer meta-package is synchronized.")
|
||||||
|
return 0
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(expected, encoding="utf-8")
|
||||||
|
print(f"Developer meta-package written to {output}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate the exact registry package set for a GovOPlaN runtime release."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
NAME = re.compile(r"^govoplan-[a-z0-9-]+$")
|
||||||
|
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
|
||||||
|
GIT_REQUIREMENT = re.compile(
|
||||||
|
r"^(?P<package>govoplan-[a-z0-9-]+)(?:\[(?P<extras>[^]]+)\])?\s+@\s+"
|
||||||
|
r"(?P<url>git\+[^\s]+/GovOPlaN/(?P<repo>govoplan-[a-z0-9-]+)\.git@v"
|
||||||
|
r"(?P<version>[A-Za-z0-9._+!-]+))$"
|
||||||
|
)
|
||||||
|
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
|
||||||
|
|
||||||
|
|
||||||
|
class PackageSetError(ValueError):
|
||||||
|
"""Release source references cannot form an immutable package set."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"--version",
|
||||||
|
help="Core/meta release version. Defaults to the workspace Core version.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--requirements",
|
||||||
|
type=Path,
|
||||||
|
default=META_ROOT / "requirements-release.txt",
|
||||||
|
)
|
||||||
|
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||||
|
parser.add_argument("--output", type=Path, required=True)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
|
||||||
|
values: list[dict[str, object]] = []
|
||||||
|
for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||||
|
line = raw.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
local = LOCAL_CORE.fullmatch(line)
|
||||||
|
if local:
|
||||||
|
values.append(
|
||||||
|
{
|
||||||
|
"name": "govoplan-core",
|
||||||
|
"version": core_version.removeprefix("v"),
|
||||||
|
"repository": "govoplan-core",
|
||||||
|
"extras": _extras(local.group("extras")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
match = GIT_REQUIREMENT.fullmatch(line)
|
||||||
|
if match is None:
|
||||||
|
raise PackageSetError(
|
||||||
|
f"unsupported release requirement at {path}:{line_number}: {line!r}"
|
||||||
|
)
|
||||||
|
values.append(
|
||||||
|
{
|
||||||
|
"name": match.group("package"),
|
||||||
|
"version": match.group("version"),
|
||||||
|
"repository": match.group("repo"),
|
||||||
|
"extras": _extras(match.group("extras")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
names = [str(item["name"]) for item in values]
|
||||||
|
if not values or names.count("govoplan-core") != 1 or len(names) != len(set(names)):
|
||||||
|
raise PackageSetError("release requirements must contain one Core and unique packages")
|
||||||
|
return tuple(values)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_package_set(
|
||||||
|
*,
|
||||||
|
core_version: str,
|
||||||
|
requirements: Path,
|
||||||
|
workspace: Path,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
core_version = core_version.removeprefix("v")
|
||||||
|
if VERSION.fullmatch(core_version) is None:
|
||||||
|
raise PackageSetError("release version is invalid")
|
||||||
|
python_packages: list[dict[str, object]] = []
|
||||||
|
webui_packages: list[dict[str, object]] = []
|
||||||
|
seen_webui: set[str] = set()
|
||||||
|
for requirement in parse_release_requirements(requirements, core_version=core_version):
|
||||||
|
repository = workspace / str(requirement["repository"])
|
||||||
|
tag = f"v{requirement['version']}"
|
||||||
|
if not (repository / ".git").is_dir():
|
||||||
|
raise PackageSetError(f"release repository is missing: {repository}")
|
||||||
|
commit = _git(repository, "rev-list", "-n", "1", tag)
|
||||||
|
if not commit:
|
||||||
|
raise PackageSetError(f"release tag is missing: {repository.name}@{tag}")
|
||||||
|
project = tomllib.loads(_git(repository, "show", f"{tag}:pyproject.toml"))["project"]
|
||||||
|
if project.get("name") != requirement["name"] or project.get("version") != requirement["version"]:
|
||||||
|
raise PackageSetError(f"tag metadata does not match {repository.name}@{tag}")
|
||||||
|
entry = {
|
||||||
|
**requirement,
|
||||||
|
"tag": tag,
|
||||||
|
"commit": commit,
|
||||||
|
}
|
||||||
|
python_packages.append(entry)
|
||||||
|
try:
|
||||||
|
webui_raw = _git(repository, "show", f"{tag}:webui/package.json")
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
continue
|
||||||
|
webui = json.loads(webui_raw)
|
||||||
|
webui_name = webui.get("name")
|
||||||
|
if (
|
||||||
|
not isinstance(webui_name, str)
|
||||||
|
or not webui_name.startswith("@govoplan/")
|
||||||
|
or webui.get("version") != requirement["version"]
|
||||||
|
or webui_name in seen_webui
|
||||||
|
):
|
||||||
|
raise PackageSetError(f"WebUI tag metadata does not match {repository.name}@{tag}")
|
||||||
|
seen_webui.add(webui_name)
|
||||||
|
webui_packages.append(
|
||||||
|
{
|
||||||
|
"name": webui_name,
|
||||||
|
"version": requirement["version"],
|
||||||
|
"repository": requirement["repository"],
|
||||||
|
"tag": tag,
|
||||||
|
"commit": commit,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
payload: dict[str, object] = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"release_version": core_version,
|
||||||
|
"registries": {
|
||||||
|
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
|
||||||
|
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
|
||||||
|
},
|
||||||
|
"python": python_packages,
|
||||||
|
"webui": webui_packages,
|
||||||
|
}
|
||||||
|
payload["package_set_sha256"] = _canonical_sha256(payload)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _extras(value: str | None) -> list[str]:
|
||||||
|
if not value:
|
||||||
|
return []
|
||||||
|
extras = sorted({item.strip() for item in value.split(",") if item.strip()})
|
||||||
|
if any(re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None for item in extras):
|
||||||
|
raise PackageSetError("release requirement contains an invalid extra")
|
||||||
|
return extras
|
||||||
|
|
||||||
|
|
||||||
|
def _git(repository: Path, *arguments: str) -> str:
|
||||||
|
return subprocess.check_output(
|
||||||
|
["git", "-C", str(repository), *arguments],
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_sha256(value: object) -> str:
|
||||||
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
return hashlib.sha256(encoded).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
try:
|
||||||
|
workspace = args.workspace.expanduser().resolve()
|
||||||
|
version = args.version
|
||||||
|
if not version:
|
||||||
|
core = tomllib.loads(
|
||||||
|
(workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
version = str(core["project"]["version"])
|
||||||
|
payload = generate_package_set(
|
||||||
|
core_version=version,
|
||||||
|
requirements=args.requirements.expanduser().resolve(),
|
||||||
|
workspace=workspace,
|
||||||
|
)
|
||||||
|
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
|
||||||
|
print(f"error: {exc}")
|
||||||
|
return 1
|
||||||
|
output = args.output.expanduser()
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
print(f"Release package set written to {output}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -8,6 +8,9 @@ WEBUI_DIR="${1:-$CORE_ROOT/webui}"
|
|||||||
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
|
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
|
||||||
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
|
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
|
||||||
export GOVOPLAN_DEPS
|
export GOVOPLAN_DEPS
|
||||||
|
PACKAGE_LOCK="${GOVOPLAN_WEBUI_PACKAGE_LOCK:-}"
|
||||||
|
PACKAGE_DIR="${GOVOPLAN_WEBUI_PACKAGE_DIR:-}"
|
||||||
|
PYTHON_BIN="${PYTHON:-python3}"
|
||||||
|
|
||||||
trap 'rm -rf "$WORK_ROOT"' EXIT
|
trap 'rm -rf "$WORK_ROOT"' EXIT
|
||||||
|
|
||||||
@@ -55,9 +58,69 @@ rm -f package-lock.json
|
|||||||
npm cache clean --force
|
npm cache clean --force
|
||||||
retry npm install --prefer-online
|
retry npm install --prefer-online
|
||||||
|
|
||||||
|
if [[ -n "$PACKAGE_LOCK" || -n "$PACKAGE_DIR" ]]; then
|
||||||
|
[[ -n "$PACKAGE_LOCK" && -n "$PACKAGE_DIR" ]] || {
|
||||||
|
echo "GOVOPLAN_WEBUI_PACKAGE_LOCK and GOVOPLAN_WEBUI_PACKAGE_DIR must be set together" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
"$PYTHON_BIN" - "$PACKAGE_LOCK" "$PACKAGE_DIR" "$GOVOPLAN_DEPS" <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
lock_path = Path(sys.argv[1]).resolve()
|
||||||
|
package_dir = Path(sys.argv[2]).resolve()
|
||||||
|
output = Path(sys.argv[3])
|
||||||
|
lock = json.loads(lock_path.read_text(encoding="utf-8"))
|
||||||
|
if lock.get("schema_version") != "1" or not isinstance(lock.get("webui"), list):
|
||||||
|
raise SystemExit("WebUI package lock is malformed")
|
||||||
|
unsigned = dict(lock)
|
||||||
|
expected_lock_hash = unsigned.pop("lock_sha256", None)
|
||||||
|
actual_lock_hash = hashlib.sha256(
|
||||||
|
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
if expected_lock_hash != actual_lock_hash:
|
||||||
|
raise SystemExit("WebUI package lock hash does not match its contents")
|
||||||
|
rows = {}
|
||||||
|
for item in lock["webui"]:
|
||||||
|
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
|
||||||
|
raise SystemExit("WebUI package lock contains a malformed artifact")
|
||||||
|
if item["name"] in rows:
|
||||||
|
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
|
||||||
|
rows[item["name"]] = item
|
||||||
|
requested = []
|
||||||
|
for line in output.read_text(encoding="utf-8").splitlines():
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
name, _source_ref = line.split("\t", 1)
|
||||||
|
row = rows.get(name)
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
raise SystemExit(f"WebUI package lock has no artifact for {name}")
|
||||||
|
filename = row.get("filename")
|
||||||
|
if not isinstance(filename, str) or Path(filename).name != filename:
|
||||||
|
raise SystemExit(f"WebUI package lock has an invalid filename for {name}")
|
||||||
|
artifact = package_dir / filename
|
||||||
|
if artifact.is_symlink() or not artifact.is_file():
|
||||||
|
raise SystemExit(f"WebUI package artifact is missing for {name}")
|
||||||
|
encoded = artifact.read_bytes()
|
||||||
|
if len(encoded) != row.get("size") or hashlib.sha256(encoded).hexdigest() != row.get("sha256"):
|
||||||
|
raise SystemExit(f"WebUI package artifact hash does not match for {name}")
|
||||||
|
requested.append(f"{name}\tfile:{artifact}")
|
||||||
|
output.write_text("\n".join(requested) + "\n", encoding="utf-8")
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
|
||||||
module_paths=()
|
module_paths=()
|
||||||
while IFS=$'\t' read -r package_name spec; do
|
while IFS=$'\t' read -r package_name spec; do
|
||||||
[[ -n "${package_name:-}" ]] || continue
|
[[ -n "${package_name:-}" ]] || continue
|
||||||
|
if [[ "$spec" == file:* ]]; then
|
||||||
|
echo "Installing $package_name from verified package artifact"
|
||||||
|
module_paths+=("$spec")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
git_url="${spec%%#*}"
|
git_url="${spec%%#*}"
|
||||||
git_ref="${spec#*#}"
|
git_ref="${spec#*#}"
|
||||||
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
|
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
|
||||||
|
|||||||
@@ -456,6 +456,13 @@ path.write_text(updated)
|
|||||||
PYCODE
|
PYCODE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
update_developer_meta_package() {
|
||||||
|
"$PYTHON" "$META_ROOT/tools/release/generate-developer-meta-package.py" \
|
||||||
|
--workspace "$PARENT" \
|
||||||
|
--requirements "$META_ROOT/requirements-release.txt" \
|
||||||
|
--output "$META_ROOT/packages/govoplan-meta/pyproject.toml"
|
||||||
|
}
|
||||||
|
|
||||||
update_version_files() {
|
update_version_files() {
|
||||||
local repo="$1"
|
local repo="$1"
|
||||||
local version="$2"
|
local version="$2"
|
||||||
@@ -875,8 +882,10 @@ done
|
|||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
|
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
|
||||||
|
echo "Would synchronize packages/govoplan-meta/pyproject.toml"
|
||||||
else
|
else
|
||||||
update_release_requirements "$TARGET_VERSION"
|
update_release_requirements "$TARGET_VERSION"
|
||||||
|
update_developer_meta_package
|
||||||
fi
|
fi
|
||||||
|
|
||||||
refresh_development_webui_lock
|
refresh_development_webui_lock
|
||||||
|
|||||||
@@ -0,0 +1,361 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Download, verify, and lock exact GovOPlaN registry package artifacts."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
from email.parser import BytesParser
|
||||||
|
from email.policy import compat32
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
from urllib.parse import quote, urlsplit, urlunsplit
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
|
||||||
|
NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||||
|
WEBUI_NAME = re.compile(r"^@govoplan/[a-z0-9]+(?:-[a-z0-9]+)*-webui$")
|
||||||
|
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
|
||||||
|
COMMIT = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
MAX_ARTIFACT_BYTES = 512 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class PackageArtifactError(ValueError):
|
||||||
|
"""Registry artifacts do not match the selected package set."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--package-set", type=Path, required=True)
|
||||||
|
parser.add_argument("--wheelhouse", type=Path, required=True)
|
||||||
|
parser.add_argument("--webui-packages", type=Path, required=True)
|
||||||
|
parser.add_argument("--lock-output", type=Path, required=True)
|
||||||
|
parser.add_argument("--requirements-output", type=Path)
|
||||||
|
parser.add_argument("--python", default=sys.executable)
|
||||||
|
parser.add_argument("--npm", default="npm")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(args: argparse.Namespace) -> dict[str, object]:
|
||||||
|
package_set = _load_package_set(args.package_set)
|
||||||
|
wheelhouse = args.wheelhouse.expanduser().resolve()
|
||||||
|
webui_packages = args.webui_packages.expanduser().resolve()
|
||||||
|
_require_empty_destination(wheelhouse)
|
||||||
|
_require_empty_destination(webui_packages)
|
||||||
|
wheelhouse.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
webui_packages.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-package-resolution-") as value:
|
||||||
|
temporary = Path(value)
|
||||||
|
wheels = temporary / "wheels"
|
||||||
|
webui = temporary / "webui"
|
||||||
|
wheels.mkdir()
|
||||||
|
webui.mkdir()
|
||||||
|
_download_wheels(
|
||||||
|
packages=tuple(package_set["python"]),
|
||||||
|
destination=wheels,
|
||||||
|
python=args.python,
|
||||||
|
index_url=str(package_set["registries"]["python"]),
|
||||||
|
)
|
||||||
|
_download_webui(
|
||||||
|
packages=tuple(package_set["webui"]),
|
||||||
|
destination=webui,
|
||||||
|
npm=args.npm,
|
||||||
|
registry=str(package_set["registries"]["npm"]),
|
||||||
|
)
|
||||||
|
python_rows = _verify_wheels(tuple(package_set["python"]), wheels)
|
||||||
|
webui_rows = _verify_webui(tuple(package_set["webui"]), webui)
|
||||||
|
lock: dict[str, object] = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"release_version": package_set["release_version"],
|
||||||
|
"package_set_sha256": package_set["package_set_sha256"],
|
||||||
|
"registries": package_set["registries"],
|
||||||
|
"python": python_rows,
|
||||||
|
"webui": webui_rows,
|
||||||
|
}
|
||||||
|
lock["lock_sha256"] = _canonical_sha256(lock)
|
||||||
|
shutil.copytree(wheels, wheelhouse, dirs_exist_ok=True)
|
||||||
|
shutil.copytree(webui, webui_packages, dirs_exist_ok=True)
|
||||||
|
args.lock_output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.lock_output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
if args.requirements_output is not None:
|
||||||
|
_write_requirements(args.requirements_output, python_rows)
|
||||||
|
return lock
|
||||||
|
|
||||||
|
|
||||||
|
def _load_package_set(path: Path) -> dict[str, object]:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(value, dict) or value.get("schema_version") != "1":
|
||||||
|
raise PackageArtifactError("package set has an unsupported shape")
|
||||||
|
expected_hash = value.get("package_set_sha256")
|
||||||
|
unsigned = dict(value)
|
||||||
|
unsigned.pop("package_set_sha256", None)
|
||||||
|
if expected_hash != _canonical_sha256(unsigned):
|
||||||
|
raise PackageArtifactError("package set hash does not match its contents")
|
||||||
|
registries = value.get("registries")
|
||||||
|
if not isinstance(registries, dict) or set(registries) != {"python", "npm"}:
|
||||||
|
raise PackageArtifactError("package set registries are invalid")
|
||||||
|
for registry in registries.values():
|
||||||
|
parsed = urlsplit(str(registry))
|
||||||
|
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||||
|
raise PackageArtifactError("package registries must use credential-free HTTPS URLs")
|
||||||
|
for group in ("python", "webui"):
|
||||||
|
packages = value.get(group)
|
||||||
|
if not isinstance(packages, list) or not packages:
|
||||||
|
raise PackageArtifactError(f"package set {group} entries are missing")
|
||||||
|
_validate_package_entries(group, packages)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_package_entries(group: str, packages: list[object]) -> None:
|
||||||
|
names: set[str] = set()
|
||||||
|
for raw in packages:
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise PackageArtifactError(f"package set {group} entry is malformed")
|
||||||
|
name = raw.get("name")
|
||||||
|
version = raw.get("version")
|
||||||
|
repository = raw.get("repository")
|
||||||
|
tag = raw.get("tag")
|
||||||
|
commit = raw.get("commit")
|
||||||
|
name_valid = (
|
||||||
|
isinstance(name, str)
|
||||||
|
and (NAME.fullmatch(name) if group == "python" else WEBUI_NAME.fullmatch(name))
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not name_valid
|
||||||
|
or name in names
|
||||||
|
or not isinstance(version, str)
|
||||||
|
or VERSION.fullmatch(version) is None
|
||||||
|
or not isinstance(repository, str)
|
||||||
|
or NAME.fullmatch(repository) is None
|
||||||
|
or tag != f"v{version}"
|
||||||
|
or not isinstance(commit, str)
|
||||||
|
or COMMIT.fullmatch(commit) is None
|
||||||
|
):
|
||||||
|
raise PackageArtifactError(f"package set {group} entry has an invalid identity")
|
||||||
|
names.add(name)
|
||||||
|
if group == "python":
|
||||||
|
extras = raw.get("extras")
|
||||||
|
if not isinstance(extras, list) or any(
|
||||||
|
not isinstance(item, str)
|
||||||
|
or re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None
|
||||||
|
for item in extras
|
||||||
|
):
|
||||||
|
raise PackageArtifactError("package set Python extras are invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _download_wheels(
|
||||||
|
*, packages: tuple[dict[str, object], ...], destination: Path, python: str, index_url: str
|
||||||
|
) -> None:
|
||||||
|
requirements = [_python_requirement(item) for item in packages]
|
||||||
|
environment = dict(os.environ)
|
||||||
|
environment["PIP_INDEX_URL"] = _authenticated_url(index_url)
|
||||||
|
environment["PIP_EXTRA_INDEX_URL"] = ""
|
||||||
|
environment["PIP_CONFIG_FILE"] = os.devnull
|
||||||
|
environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
||||||
|
subprocess.run(
|
||||||
|
[python, "-m", "pip", "download", "--no-deps", "--only-binary=:all:", "--dest", str(destination), *requirements],
|
||||||
|
check=True,
|
||||||
|
env=environment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _download_webui(
|
||||||
|
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
|
||||||
|
) -> None:
|
||||||
|
environment = dict(os.environ)
|
||||||
|
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
|
||||||
|
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
|
||||||
|
if token:
|
||||||
|
parsed = urlsplit(registry)
|
||||||
|
auth_path = f"//{parsed.netloc}{parsed.path}:_authToken={token}\n"
|
||||||
|
npmrc = tempfile.NamedTemporaryFile(prefix="govoplan-npmrc-", delete=False)
|
||||||
|
npmrc.write(f"@govoplan:registry={registry}\n{auth_path}".encode("utf-8"))
|
||||||
|
npmrc.close()
|
||||||
|
os.chmod(npmrc.name, 0o600)
|
||||||
|
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
|
||||||
|
try:
|
||||||
|
for item in packages:
|
||||||
|
subprocess.run(
|
||||||
|
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
|
||||||
|
check=True,
|
||||||
|
env=environment,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if npmrc is not None:
|
||||||
|
Path(npmrc.name).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||||
|
expected = {_normalize(str(item["name"])): item for item in packages}
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for path in sorted(root.glob("*.whl")):
|
||||||
|
identity = _wheel_identity(path)
|
||||||
|
name = str(identity["name"])
|
||||||
|
package = expected.get(name)
|
||||||
|
if package is None or identity["version"] != package["version"] or name in seen:
|
||||||
|
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
|
||||||
|
seen.add(name)
|
||||||
|
rows.append(_artifact_row(path, package))
|
||||||
|
if seen != set(expected):
|
||||||
|
raise PackageArtifactError("registry did not return every selected Python wheel")
|
||||||
|
return sorted(rows, key=lambda item: str(item["name"]))
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||||
|
expected = {str(item["name"]): item for item in packages}
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for path in sorted(root.glob("*.tgz")):
|
||||||
|
identity = _npm_identity(path)
|
||||||
|
name = str(identity["name"])
|
||||||
|
package = expected.get(name)
|
||||||
|
if package is None or identity["version"] != package["version"] or name in seen:
|
||||||
|
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
|
||||||
|
seen.add(name)
|
||||||
|
row = _artifact_row(path, package)
|
||||||
|
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
|
||||||
|
rows.append(row)
|
||||||
|
if seen != set(expected):
|
||||||
|
raise PackageArtifactError("registry did not return every selected WebUI package")
|
||||||
|
return sorted(rows, key=lambda item: str(item["name"]))
|
||||||
|
|
||||||
|
|
||||||
|
def _wheel_identity(path: Path) -> dict[str, str]:
|
||||||
|
_bounded(path)
|
||||||
|
with zipfile.ZipFile(path) as archive:
|
||||||
|
metadata = [
|
||||||
|
item for item in archive.infolist()
|
||||||
|
if PurePosixPath(item.filename).name == "METADATA"
|
||||||
|
and PurePosixPath(item.filename).parent.name.endswith(".dist-info")
|
||||||
|
]
|
||||||
|
if len(metadata) != 1 or metadata[0].file_size > 1024 * 1024:
|
||||||
|
raise PackageArtifactError(f"wheel metadata is invalid: {path.name}")
|
||||||
|
parsed = BytesParser(policy=compat32).parsebytes(archive.read(metadata[0]))
|
||||||
|
name = _normalize(str(parsed.get("Name") or ""))
|
||||||
|
version = str(parsed.get("Version") or "")
|
||||||
|
if NAME.fullmatch(name) is None or VERSION.fullmatch(version) is None:
|
||||||
|
raise PackageArtifactError(f"wheel identity is invalid: {path.name}")
|
||||||
|
return {"name": name, "version": version}
|
||||||
|
|
||||||
|
|
||||||
|
def _npm_identity(path: Path) -> dict[str, str]:
|
||||||
|
_bounded(path)
|
||||||
|
with tarfile.open(path, mode="r:gz") as archive:
|
||||||
|
try:
|
||||||
|
member = archive.getmember("package/package.json")
|
||||||
|
except KeyError as exc:
|
||||||
|
raise PackageArtifactError(f"npm package metadata is missing: {path.name}") from exc
|
||||||
|
if not member.isfile() or member.size > 1024 * 1024:
|
||||||
|
raise PackageArtifactError(f"npm package metadata is invalid: {path.name}")
|
||||||
|
extracted = archive.extractfile(member)
|
||||||
|
if extracted is None:
|
||||||
|
raise PackageArtifactError(f"npm package metadata cannot be read: {path.name}")
|
||||||
|
value = json.load(extracted)
|
||||||
|
name = value.get("name")
|
||||||
|
version = value.get("version")
|
||||||
|
if not isinstance(name, str) or not name.startswith("@govoplan/") or not isinstance(version, str) or VERSION.fullmatch(version) is None:
|
||||||
|
raise PackageArtifactError(f"npm package identity is invalid: {path.name}")
|
||||||
|
return {"name": name, "version": version}
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
|
||||||
|
row = {
|
||||||
|
"name": package["name"],
|
||||||
|
"version": package["version"],
|
||||||
|
"repository": package["repository"],
|
||||||
|
"tag": package["tag"],
|
||||||
|
"commit": package["commit"],
|
||||||
|
"filename": path.name,
|
||||||
|
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||||
|
"size": path.stat().st_size,
|
||||||
|
}
|
||||||
|
if "extras" in package:
|
||||||
|
row["extras"] = package["extras"]
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
|
||||||
|
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
|
||||||
|
for row in rows:
|
||||||
|
selected_extras = row.get("extras") or []
|
||||||
|
extras = (
|
||||||
|
f"[{','.join(str(value) for value in selected_extras)}]"
|
||||||
|
if selected_extras
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
lines.append(
|
||||||
|
f"{row['name']}{extras}=={row['version']} "
|
||||||
|
f"--hash=sha256:{row['sha256']}"
|
||||||
|
)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _python_requirement(item: dict[str, object]) -> str:
|
||||||
|
extras = item.get("extras") or []
|
||||||
|
suffix = f"[{','.join(str(value) for value in extras)}]" if extras else ""
|
||||||
|
return f"{item['name']}{suffix}=={item['version']}"
|
||||||
|
|
||||||
|
|
||||||
|
def _authenticated_url(url: str) -> str:
|
||||||
|
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
|
||||||
|
username = os.environ.get("GOVOPLAN_PACKAGE_USERNAME", "")
|
||||||
|
if not token:
|
||||||
|
return url
|
||||||
|
if not username:
|
||||||
|
raise PackageArtifactError("GOVOPLAN_PACKAGE_USERNAME is required with a package token")
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
return urlunsplit(
|
||||||
|
(
|
||||||
|
parsed.scheme,
|
||||||
|
f"{quote(username, safe='')}:{quote(token, safe='')}@{parsed.netloc}",
|
||||||
|
parsed.path,
|
||||||
|
parsed.query,
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_empty_destination(path: Path) -> None:
|
||||||
|
if path.exists() and (not path.is_dir() or any(path.iterdir())):
|
||||||
|
raise PackageArtifactError(f"output directory must be absent or empty: {path}")
|
||||||
|
if path.is_symlink():
|
||||||
|
raise PackageArtifactError(f"output directory must not be a symlink: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded(path: Path) -> None:
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_ARTIFACT_BYTES:
|
||||||
|
raise PackageArtifactError(f"package artifact is invalid or too large: {path.name}")
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize(value: str) -> str:
|
||||||
|
return re.sub(r"[-_.]+", "-", value.strip().lower())
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_sha256(value: object) -> str:
|
||||||
|
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
try:
|
||||||
|
lock = resolve(args)
|
||||||
|
except (PackageArtifactError, OSError, ValueError, subprocess.CalledProcessError, zipfile.BadZipFile, tarfile.TarError) as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(f"Resolved {len(lock['python'])} Python and {len(lock['webui'])} WebUI packages.")
|
||||||
|
print(f"Package artifact lock written to {args.lock_output}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Install or verify the canonical package-release workflow in module repos."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
TEMPLATE = META_ROOT / "tools" / "repo" / "templates" / "module-package-release.yml"
|
||||||
|
DESTINATION = Path(".gitea/workflows/module-package-release.yml")
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument("--check", action="store_true")
|
||||||
|
mode.add_argument("--write", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--parent",
|
||||||
|
type=Path,
|
||||||
|
default=META_ROOT.parent,
|
||||||
|
help="Parent directory containing the repositories.",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def package_repositories(parent: Path) -> tuple[Path, ...]:
|
||||||
|
inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
|
||||||
|
repositories: list[Path] = []
|
||||||
|
for item in inventory["repositories"]:
|
||||||
|
name = str(item["name"])
|
||||||
|
if not name.startswith("govoplan-"):
|
||||||
|
continue
|
||||||
|
repository = parent / str(item["path"])
|
||||||
|
if (repository / "pyproject.toml").is_file():
|
||||||
|
repositories.append(repository)
|
||||||
|
return tuple(sorted(repositories))
|
||||||
|
|
||||||
|
|
||||||
|
def synchronize(*, parent: Path, write: bool) -> tuple[str, ...]:
|
||||||
|
expected = TEMPLATE.read_bytes()
|
||||||
|
mismatches: list[str] = []
|
||||||
|
for repository in package_repositories(parent):
|
||||||
|
destination = repository / DESTINATION
|
||||||
|
current = destination.read_bytes() if destination.is_file() else None
|
||||||
|
if current == expected:
|
||||||
|
continue
|
||||||
|
mismatches.append(repository.name)
|
||||||
|
if write:
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = destination.with_suffix(destination.suffix + ".tmp")
|
||||||
|
temporary.write_bytes(expected)
|
||||||
|
temporary.chmod(0o644)
|
||||||
|
temporary.replace(destination)
|
||||||
|
return tuple(mismatches)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
mismatches = synchronize(parent=args.parent.expanduser().resolve(), write=args.write)
|
||||||
|
if args.write:
|
||||||
|
print(f"Installed package-release workflow in {len(mismatches)} repositories.")
|
||||||
|
return 0
|
||||||
|
if mismatches:
|
||||||
|
print("Package-release workflow is missing or stale in:", file=sys.stderr)
|
||||||
|
for repository in mismatches:
|
||||||
|
print(f"- {repository}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print("Package-release workflows are synchronized.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
name: Module Package Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
release_tag:
|
||||||
|
description: Existing protected version tag to publish
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-packages:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Select and validate protected release tag
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_TAG: ${{ inputs.release_tag }}
|
||||||
|
TRIGGER_TAG: ${{ gitea.ref_name }}
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
|
||||||
|
case "$tag" in
|
||||||
|
v[0-9]*.[0-9]*.[0-9]*) ;;
|
||||||
|
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
|
||||||
|
tag_commit="$(git rev-list -n 1 "$tag")"
|
||||||
|
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
|
||||||
|
echo "Release tag is not contained in main" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
python - "$tag" <<'PY'
|
||||||
|
import fnmatch
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
tag = sys.argv[1]
|
||||||
|
repository = os.environ["GITEA_REPOSITORY"]
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"{os.environ['GITEA_API_URL']}/repos/{repository}/tag_protections",
|
||||||
|
headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request, timeout=30) as response:
|
||||||
|
protections = json.load(response)
|
||||||
|
if not any(fnmatch.fnmatchcase(tag, item.get("name_pattern", "")) for item in protections):
|
||||||
|
raise SystemExit(f"Release tag {tag!r} is not covered by repository tag protection")
|
||||||
|
PY
|
||||||
|
git checkout --detach "$tag"
|
||||||
|
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
|
||||||
|
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
|
||||||
|
- name: Validate package versions
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
tag = os.environ["RELEASE_TAG"]
|
||||||
|
expected = tag.removeprefix("v")
|
||||||
|
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||||
|
if project.get("version") != expected:
|
||||||
|
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
|
||||||
|
raise SystemExit("Python distribution name must use the govoplan-* namespace")
|
||||||
|
webui = Path("webui/package.json")
|
||||||
|
if webui.is_file():
|
||||||
|
package = json.loads(webui.read_text(encoding="utf-8"))
|
||||||
|
if package.get("version") != expected:
|
||||||
|
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
|
||||||
|
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
|
||||||
|
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
|
||||||
|
release = Path("webui/package.release.json")
|
||||||
|
if release.is_file():
|
||||||
|
release_package = json.loads(release.read_text(encoding="utf-8"))
|
||||||
|
if (
|
||||||
|
release_package.get("name") != package.get("name")
|
||||||
|
or release_package.get("version") != expected
|
||||||
|
):
|
||||||
|
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
|
||||||
|
PY
|
||||||
|
- name: Build immutable package artifacts
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
|
||||||
|
rm -rf dist .package-webui
|
||||||
|
python -m build --wheel --outdir dist
|
||||||
|
python -m twine check dist/*.whl
|
||||||
|
if [[ -f webui/package.json ]]; then
|
||||||
|
mkdir .package-webui
|
||||||
|
cp -a webui/. .package-webui/
|
||||||
|
rm -rf .package-webui/node_modules .package-webui/dist
|
||||||
|
if [[ -f .package-webui/package.release.json ]]; then
|
||||||
|
cp .package-webui/package.release.json .package-webui/package.json
|
||||||
|
fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = ".package-webui/package.json";
|
||||||
|
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||||
|
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
|
||||||
|
if (!name.startsWith("@govoplan/")) continue;
|
||||||
|
if (typeof specifier !== "string") {
|
||||||
|
throw new Error(`${group}.${name} must use a string version`);
|
||||||
|
}
|
||||||
|
const packageSlug = name.slice("@govoplan/".length);
|
||||||
|
if (!packageSlug.endsWith("-webui")) {
|
||||||
|
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
|
||||||
|
}
|
||||||
|
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
|
||||||
|
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
const gitTag = specifier.match(
|
||||||
|
new RegExp(
|
||||||
|
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/GovOPlaN/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (gitTag) {
|
||||||
|
packageJson[group][name] = gitTag[1];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
|
||||||
|
throw new Error(
|
||||||
|
`${group}.${name} must resolve to an exact registry version for publication`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete packageJson.private;
|
||||||
|
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
|
||||||
|
NODE
|
||||||
|
npm pkg delete private --prefix .package-webui
|
||||||
|
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
|
||||||
|
fi
|
||||||
|
python - <<'PY'
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
artifacts = []
|
||||||
|
for path in sorted(Path("dist").iterdir()):
|
||||||
|
if path.suffix not in {".whl", ".tgz"}:
|
||||||
|
continue
|
||||||
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
|
||||||
|
payload = {
|
||||||
|
"schema_version": "1",
|
||||||
|
"repository": os.environ["GITEA_REPOSITORY"],
|
||||||
|
"tag": os.environ["RELEASE_TAG"],
|
||||||
|
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
Path("dist/package-artifacts.json").write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
- name: Retain package hash evidence
|
||||||
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
|
||||||
|
with:
|
||||||
|
name: module-packages-${{ gitea.ref_name }}
|
||||||
|
path: dist/package-artifacts.json
|
||||||
|
- name: Publish wheel and WebUI package
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test -n "$PACKAGE_USERNAME"
|
||||||
|
test -n "$PACKAGE_TOKEN"
|
||||||
|
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
|
||||||
|
python -m twine upload --non-interactive \
|
||||||
|
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
|
||||||
|
dist/*.whl
|
||||||
|
shopt -s nullglob
|
||||||
|
webui_packages=(dist/*.tgz)
|
||||||
|
if (( ${#webui_packages[@]} )); then
|
||||||
|
npmrc="$(mktemp)"
|
||||||
|
trap 'rm -f "$npmrc"' EXIT
|
||||||
|
chmod 600 "$npmrc"
|
||||||
|
printf '%s\n' \
|
||||||
|
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
|
||||||
|
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
|
||||||
|
> "$npmrc"
|
||||||
|
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "${webui_packages[0]}" \
|
||||||
|
--ignore-scripts --access public \
|
||||||
|
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user