Add registry-backed module package releases
Dependency Audit / dependency-audit (push) Failing after 1m37s
Deployment Installer / deployment-installer (push) Successful in 6s
Security Audit / security-audit (push) Successful in 10m21s

This commit is contained in:
2026-08-04 04:14:28 +02:00
parent 1a68565ba0
commit f7590a7b8b
25 changed files with 1887 additions and 5 deletions
+3
View File
@@ -40,6 +40,9 @@ GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash
PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$META_ROOT/tools/checks/check-manifest-shapes.py" --require-architecture
cd "$META_ROOT"
"$PYTHON" tools/repo/sync-module-package-workflows.py --check
"$PYTHON" tools/release/generate-developer-meta-package.py --check
"$PYTHON" -m unittest tests.test_module_package_workflows tests.test_package_registry_release
"$PYTHON" -m unittest tests.test_deployment_installer
"$PYTHON" -m unittest tests.test_capability_fit_evidence
"$PYTHON" -m unittest tests.test_configuration_package_artifacts
@@ -167,6 +167,7 @@ def validate_manifest(
"composition",
"signatures",
},
optional={"package_lock"},
label="distribution manifest",
)
if payload.get("schema_version") != "1":
@@ -200,6 +201,12 @@ def validate_manifest(
_https_url(deployer.get("url"), "deployer.url")
_sha256(deployer.get("sha256"), "deployer.sha256")
if "package_lock" in payload:
package_lock = _object(payload.get("package_lock"), "package_lock")
_exact_keys(package_lock, required={"url", "sha256"}, label="package_lock")
_https_url(package_lock.get("url"), "package_lock.url")
_sha256(package_lock.get("sha256"), "package_lock.sha256")
images = _object(payload.get("images"), "images")
if set(images) != {"api", "web"}:
raise DistributionError("images must contain exactly api and web")
@@ -630,10 +637,12 @@ def _exact_keys(
value: Mapping[str, Any],
*,
required: set[str],
optional: set[str] | None = None,
label: str,
) -> None:
optional = optional or set()
missing = sorted(required - set(value))
extra = sorted(set(value) - required)
extra = sorted(set(value) - required - optional)
if missing or extra:
detail = []
if missing:
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Configure and verify protected GovOPlaN package-release boundaries."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
from gitea_common import (
GiteaClient,
GiteaError,
RepoTarget,
load_dotenv,
org_path,
quote_path,
repo_path,
require_token,
)
META_ROOT = Path(__file__).resolve().parents[2]
REQUIRED_SECRETS = {"GOVOPLAN_PACKAGE_USERNAME", "GOVOPLAN_PACKAGE_TOKEN"}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--url", default="https://git.add-ideas.de")
parser.add_argument("--owner", default="GovOPlaN")
parser.add_argument("--team", default="Owners")
parser.add_argument("--pattern", default="v*")
parser.add_argument("--env-file", type=Path)
parser.add_argument("--apply", action="store_true")
return parser
def package_repositories() -> tuple[str, ...]:
inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
values = ["govoplan"]
for item in inventory["repositories"]:
name = str(item["name"])
repository = META_ROOT.parent / str(item["path"])
if name.startswith("govoplan-") and (repository / "pyproject.toml").is_file():
values.append(name)
return tuple(sorted(set(values)))
def configure(
client: GiteaClient,
*,
owner: str,
team: str,
pattern: str,
apply: bool,
) -> tuple[str, ...]:
missing: list[str] = []
expected = {
"name_pattern": pattern,
"whitelist_teams": [team],
"whitelist_usernames": [],
}
for repository in package_repositories():
path = repo_path(owner, repository, "/tag_protections")
protections = client.request_json("GET", path)
matching = [
item
for item in protections
if isinstance(item, dict) and item.get("name_pattern") == pattern
]
if len(matching) == 1 and _matches(matching[0], expected):
print(f"protected {repository}:{pattern}")
continue
missing.append(repository)
if not apply:
print(f"would protect {repository}:{pattern}")
continue
if len(matching) == 1:
protection_id = matching[0].get("id")
client.request_json(
"PATCH",
f"{path}/{quote_path(str(protection_id))}",
body=expected,
)
print(f"updated {repository}:{pattern}")
elif not matching:
client.request_json("POST", path, body=expected)
print(f"created {repository}:{pattern}")
else:
raise GiteaError(f"{repository} has duplicate {pattern!r} tag protections")
return tuple(missing)
def _matches(value: dict[str, object], expected: dict[str, object]) -> bool:
return (
value.get("name_pattern") == expected["name_pattern"]
and sorted(value.get("whitelist_teams") or []) == expected["whitelist_teams"]
and sorted(value.get("whitelist_usernames") or []) == expected["whitelist_usernames"]
)
def main() -> int:
args = build_parser().parse_args()
try:
load_dotenv(args.env_file)
token = require_token()
target = RepoTarget(base_url=args.url, owner=args.owner, repo="govoplan")
with GiteaClient(target, token) as client:
mismatches = configure(
client,
owner=args.owner,
team=args.team,
pattern=args.pattern,
apply=args.apply,
)
secrets = client.request_json(
"GET", org_path(args.owner, "/actions/secrets"), query={"limit": 50}
)
names = {
str(item.get("name") or "")
for item in secrets
if isinstance(item, dict)
}
missing_secrets = sorted(REQUIRED_SECRETS - names)
if missing_secrets:
print(
"Missing organization Actions secrets: " + ", ".join(missing_secrets),
file=sys.stderr,
)
unresolved = (bool(mismatches) and not args.apply) or bool(missing_secrets)
if unresolved:
return 1
print("Package release protection and credential names are configured.")
return 0
except (GiteaError, OSError, ValueError, json.JSONDecodeError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -26,6 +26,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--web-metadata", type=Path, required=True)
parser.add_argument("--deployer", type=Path, required=True)
parser.add_argument("--deployer-url", required=True)
parser.add_argument("--package-lock", type=Path, required=True)
parser.add_argument("--artifact-base-url", required=True)
parser.add_argument("--source-commit", required=True)
parser.add_argument("--version", required=True)
@@ -45,6 +46,11 @@ def build_parser() -> argparse.ArgumentParser:
def finalize(args: argparse.Namespace) -> dict[str, Any]:
composition = _json_object(args.composition)
_validate_package_lock(
_json_object(args.package_lock),
version=args.version,
composition=composition,
)
api = _image_metadata(_json_object(args.api_metadata), "api")
web = _image_metadata(_json_object(args.web_metadata), "web")
dependencies = dict(_dependency(value) for value in args.dependency)
@@ -99,6 +105,10 @@ def finalize(args: argparse.Namespace) -> dict[str, Any]:
"url": args.deployer_url,
"sha256": _sha256_file(args.deployer),
},
"package_lock": {
"url": f"{artifact_base}/package-artifacts.lock.json",
"sha256": _sha256_file(args.package_lock),
},
"images": {
"api": {
**api,
@@ -247,6 +257,57 @@ def _json_object(path: Path) -> dict[str, Any]:
return value
def _validate_package_lock(
lock: dict[str, Any],
*,
version: str,
composition: dict[str, Any],
) -> None:
if lock.get("schema_version") != "1" or lock.get("release_version") != version:
raise ValueError("package lock schema or release version does not match")
unsigned = dict(lock)
expected_hash = unsigned.pop("lock_sha256", None)
actual_hash = hashlib.sha256(
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if expected_hash != actual_hash:
raise ValueError("package lock hash does not match its contents")
rows = lock.get("python")
if not isinstance(rows, list):
raise ValueError("package lock Python artifacts are missing")
locked = _package_identities(rows, name_key="name")
composed = _package_identities(
composition["python"]["packages"],
name_key="package",
)
if locked != composed:
raise ValueError("package lock does not match the runtime wheel composition")
def _package_identities(
rows: list[object],
*,
name_key: str,
) -> dict[str, tuple[str, str]]:
identities: dict[str, tuple[str, str]] = {}
for row in rows:
if not isinstance(row, dict):
raise ValueError("package artifact identity is malformed")
name = row.get(name_key)
version = row.get("version")
digest = row.get("sha256")
if (
not isinstance(name, str)
or not isinstance(version, str)
or not isinstance(digest, str)
or SHA256.fullmatch(digest) is None
or name in identities
):
raise ValueError("package artifact identity is malformed or duplicated")
identities[name] = (version, digest)
return identities
def _https_url(value: str, label: str) -> str:
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Generate the optional GovOPlaN developer convenience meta-package."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import re
import tomllib
META_ROOT = Path(__file__).resolve().parents[2]
DIRECT = re.compile(r"^(govoplan-[a-z0-9-]+)(?:\[([^]]+)\])?\s+@\s+.*@v([A-Za-z0-9._+!-]+)$")
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[([^]]+)\])?$")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument(
"--requirements",
type=Path,
default=META_ROOT / "requirements-release.txt",
)
parser.add_argument(
"--output",
type=Path,
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
)
parser.add_argument("--check", action="store_true")
return parser
def render(*, workspace: Path, requirements: Path) -> str:
core = tomllib.loads((workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8"))["project"]
version = str(core["version"])
base: list[str] = []
for raw in requirements.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
local = LOCAL_CORE.fullmatch(line)
if local:
extra = f"[{local.group(1)}]" if local.group(1) else ""
base.append(f"govoplan-core{extra}=={version}")
continue
match = DIRECT.fullmatch(line)
if match is None:
raise ValueError(f"unsupported release requirement: {line!r}")
extra = f"[{match.group(2)}]" if match.group(2) else ""
base.append(f"{match.group(1)}{extra}=={match.group(3)}")
base_names = {_requirement_name(item) for item in base}
full: list[str] = []
for project_path in sorted(workspace.glob("govoplan-*/pyproject.toml")):
project = tomllib.loads(project_path.read_text(encoding="utf-8"))["project"]
name = str(project.get("name") or "")
package_version = str(project.get("version") or "")
if name.startswith("govoplan-") and name not in base_names:
full.append(f"{name}=={package_version}")
return "\n".join(
[
"[build-system]",
'requires = ["setuptools>=69", "wheel"]',
'build-backend = "setuptools.build_meta"',
"",
"[project]",
'name = "govoplan"',
f'version = {json.dumps(version)}',
'description = "Developer convenience package for a versioned GovOPlaN composition"',
'readme = "README.md"',
'requires-python = ">=3.12"',
'license = { text = "AGPL-3.0-or-later" }',
"dependencies = [",
*[f" {json.dumps(item)}," for item in base],
"]",
"",
"[project.optional-dependencies]",
"full = [",
*[f" {json.dumps(item)}," for item in full],
"]",
"",
"[project.urls]",
'Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"',
'Documentation = "https://govoplan.add-ideas.de"',
"",
"[tool.setuptools.packages.find]",
'where = ["src"]',
"",
]
)
def _requirement_name(value: str) -> str:
return value.split("[", 1)[0].split("==", 1)[0]
def main() -> int:
args = build_parser().parse_args()
try:
expected = render(
workspace=args.workspace.expanduser().resolve(),
requirements=args.requirements.expanduser().resolve(),
)
except (OSError, KeyError, ValueError, tomllib.TOMLDecodeError) as exc:
print(f"error: {exc}")
return 1
output = args.output.expanduser()
current = output.read_text(encoding="utf-8") if output.is_file() else None
if args.check:
if current != expected:
print(f"error: developer meta-package is stale: {output}")
return 1
print("Developer meta-package is synchronized.")
return 0
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(expected, encoding="utf-8")
print(f"Developer meta-package written to {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Generate the exact registry package set for a GovOPlaN runtime release."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import subprocess
import tomllib
META_ROOT = Path(__file__).resolve().parents[2]
NAME = re.compile(r"^govoplan-[a-z0-9-]+$")
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
GIT_REQUIREMENT = re.compile(
r"^(?P<package>govoplan-[a-z0-9-]+)(?:\[(?P<extras>[^]]+)\])?\s+@\s+"
r"(?P<url>git\+[^\s]+/GovOPlaN/(?P<repo>govoplan-[a-z0-9-]+)\.git@v"
r"(?P<version>[A-Za-z0-9._+!-]+))$"
)
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
class PackageSetError(ValueError):
"""Release source references cannot form an immutable package set."""
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--version",
help="Core/meta release version. Defaults to the workspace Core version.",
)
parser.add_argument(
"--requirements",
type=Path,
default=META_ROOT / "requirements-release.txt",
)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument("--output", type=Path, required=True)
return parser
def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
values: list[dict[str, object]] = []
for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
local = LOCAL_CORE.fullmatch(line)
if local:
values.append(
{
"name": "govoplan-core",
"version": core_version.removeprefix("v"),
"repository": "govoplan-core",
"extras": _extras(local.group("extras")),
}
)
continue
match = GIT_REQUIREMENT.fullmatch(line)
if match is None:
raise PackageSetError(
f"unsupported release requirement at {path}:{line_number}: {line!r}"
)
values.append(
{
"name": match.group("package"),
"version": match.group("version"),
"repository": match.group("repo"),
"extras": _extras(match.group("extras")),
}
)
names = [str(item["name"]) for item in values]
if not values or names.count("govoplan-core") != 1 or len(names) != len(set(names)):
raise PackageSetError("release requirements must contain one Core and unique packages")
return tuple(values)
def generate_package_set(
*,
core_version: str,
requirements: Path,
workspace: Path,
) -> dict[str, object]:
core_version = core_version.removeprefix("v")
if VERSION.fullmatch(core_version) is None:
raise PackageSetError("release version is invalid")
python_packages: list[dict[str, object]] = []
webui_packages: list[dict[str, object]] = []
seen_webui: set[str] = set()
for requirement in parse_release_requirements(requirements, core_version=core_version):
repository = workspace / str(requirement["repository"])
tag = f"v{requirement['version']}"
if not (repository / ".git").is_dir():
raise PackageSetError(f"release repository is missing: {repository}")
commit = _git(repository, "rev-list", "-n", "1", tag)
if not commit:
raise PackageSetError(f"release tag is missing: {repository.name}@{tag}")
project = tomllib.loads(_git(repository, "show", f"{tag}:pyproject.toml"))["project"]
if project.get("name") != requirement["name"] or project.get("version") != requirement["version"]:
raise PackageSetError(f"tag metadata does not match {repository.name}@{tag}")
entry = {
**requirement,
"tag": tag,
"commit": commit,
}
python_packages.append(entry)
try:
webui_raw = _git(repository, "show", f"{tag}:webui/package.json")
except subprocess.CalledProcessError:
continue
webui = json.loads(webui_raw)
webui_name = webui.get("name")
if (
not isinstance(webui_name, str)
or not webui_name.startswith("@govoplan/")
or webui.get("version") != requirement["version"]
or webui_name in seen_webui
):
raise PackageSetError(f"WebUI tag metadata does not match {repository.name}@{tag}")
seen_webui.add(webui_name)
webui_packages.append(
{
"name": webui_name,
"version": requirement["version"],
"repository": requirement["repository"],
"tag": tag,
"commit": commit,
}
)
payload: dict[str, object] = {
"schema_version": "1",
"release_version": core_version,
"registries": {
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
},
"python": python_packages,
"webui": webui_packages,
}
payload["package_set_sha256"] = _canonical_sha256(payload)
return payload
def _extras(value: str | None) -> list[str]:
if not value:
return []
extras = sorted({item.strip() for item in value.split(",") if item.strip()})
if any(re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None for item in extras):
raise PackageSetError("release requirement contains an invalid extra")
return extras
def _git(repository: Path, *arguments: str) -> str:
return subprocess.check_output(
["git", "-C", str(repository), *arguments],
text=True,
stderr=subprocess.DEVNULL,
).strip()
def _canonical_sha256(value: object) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def main() -> int:
args = build_parser().parse_args()
try:
workspace = args.workspace.expanduser().resolve()
version = args.version
if not version:
core = tomllib.loads(
(workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8")
)
version = str(core["project"]["version"])
payload = generate_package_set(
core_version=version,
requirements=args.requirements.expanduser().resolve(),
workspace=workspace,
)
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
print(f"error: {exc}")
return 1
output = args.output.expanduser()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"Release package set written to {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -8,6 +8,9 @@ WEBUI_DIR="${1:-$CORE_ROOT/webui}"
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
export GOVOPLAN_DEPS
PACKAGE_LOCK="${GOVOPLAN_WEBUI_PACKAGE_LOCK:-}"
PACKAGE_DIR="${GOVOPLAN_WEBUI_PACKAGE_DIR:-}"
PYTHON_BIN="${PYTHON:-python3}"
trap 'rm -rf "$WORK_ROOT"' EXIT
@@ -55,9 +58,69 @@ rm -f package-lock.json
npm cache clean --force
retry npm install --prefer-online
if [[ -n "$PACKAGE_LOCK" || -n "$PACKAGE_DIR" ]]; then
[[ -n "$PACKAGE_LOCK" && -n "$PACKAGE_DIR" ]] || {
echo "GOVOPLAN_WEBUI_PACKAGE_LOCK and GOVOPLAN_WEBUI_PACKAGE_DIR must be set together" >&2
exit 1
}
"$PYTHON_BIN" - "$PACKAGE_LOCK" "$PACKAGE_DIR" "$GOVOPLAN_DEPS" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
lock_path = Path(sys.argv[1]).resolve()
package_dir = Path(sys.argv[2]).resolve()
output = Path(sys.argv[3])
lock = json.loads(lock_path.read_text(encoding="utf-8"))
if lock.get("schema_version") != "1" or not isinstance(lock.get("webui"), list):
raise SystemExit("WebUI package lock is malformed")
unsigned = dict(lock)
expected_lock_hash = unsigned.pop("lock_sha256", None)
actual_lock_hash = hashlib.sha256(
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if expected_lock_hash != actual_lock_hash:
raise SystemExit("WebUI package lock hash does not match its contents")
rows = {}
for item in lock["webui"]:
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
raise SystemExit("WebUI package lock contains a malformed artifact")
if item["name"] in rows:
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
rows[item["name"]] = item
requested = []
for line in output.read_text(encoding="utf-8").splitlines():
if not line:
continue
name, _source_ref = line.split("\t", 1)
row = rows.get(name)
if not isinstance(row, dict):
raise SystemExit(f"WebUI package lock has no artifact for {name}")
filename = row.get("filename")
if not isinstance(filename, str) or Path(filename).name != filename:
raise SystemExit(f"WebUI package lock has an invalid filename for {name}")
artifact = package_dir / filename
if artifact.is_symlink() or not artifact.is_file():
raise SystemExit(f"WebUI package artifact is missing for {name}")
encoded = artifact.read_bytes()
if len(encoded) != row.get("size") or hashlib.sha256(encoded).hexdigest() != row.get("sha256"):
raise SystemExit(f"WebUI package artifact hash does not match for {name}")
requested.append(f"{name}\tfile:{artifact}")
output.write_text("\n".join(requested) + "\n", encoding="utf-8")
PY
fi
module_paths=()
while IFS=$'\t' read -r package_name spec; do
[[ -n "${package_name:-}" ]] || continue
if [[ "$spec" == file:* ]]; then
echo "Installing $package_name from verified package artifact"
module_paths+=("$spec")
continue
fi
git_url="${spec%%#*}"
git_ref="${spec#*#}"
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
+9
View File
@@ -456,6 +456,13 @@ path.write_text(updated)
PYCODE
}
update_developer_meta_package() {
"$PYTHON" "$META_ROOT/tools/release/generate-developer-meta-package.py" \
--workspace "$PARENT" \
--requirements "$META_ROOT/requirements-release.txt" \
--output "$META_ROOT/packages/govoplan-meta/pyproject.toml"
}
update_version_files() {
local repo="$1"
local version="$2"
@@ -875,8 +882,10 @@ done
if [[ "$DRY_RUN" -eq 1 ]]; then
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
echo "Would synchronize packages/govoplan-meta/pyproject.toml"
else
update_release_requirements "$TARGET_VERSION"
update_developer_meta_package
fi
refresh_development_webui_lock
+361
View File
@@ -0,0 +1,361 @@
#!/usr/bin/env python3
"""Download, verify, and lock exact GovOPlaN registry package artifacts."""
from __future__ import annotations
import argparse
import base64
from email.parser import BytesParser
from email.policy import compat32
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
from urllib.parse import quote, urlsplit, urlunsplit
import zipfile
NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
WEBUI_NAME = re.compile(r"^@govoplan/[a-z0-9]+(?:-[a-z0-9]+)*-webui$")
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
COMMIT = re.compile(r"^[0-9a-f]{40}$")
MAX_ARTIFACT_BYTES = 512 * 1024 * 1024
class PackageArtifactError(ValueError):
"""Registry artifacts do not match the selected package set."""
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package-set", type=Path, required=True)
parser.add_argument("--wheelhouse", type=Path, required=True)
parser.add_argument("--webui-packages", type=Path, required=True)
parser.add_argument("--lock-output", type=Path, required=True)
parser.add_argument("--requirements-output", type=Path)
parser.add_argument("--python", default=sys.executable)
parser.add_argument("--npm", default="npm")
return parser
def resolve(args: argparse.Namespace) -> dict[str, object]:
package_set = _load_package_set(args.package_set)
wheelhouse = args.wheelhouse.expanduser().resolve()
webui_packages = args.webui_packages.expanduser().resolve()
_require_empty_destination(wheelhouse)
_require_empty_destination(webui_packages)
wheelhouse.parent.mkdir(parents=True, exist_ok=True)
webui_packages.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="govoplan-package-resolution-") as value:
temporary = Path(value)
wheels = temporary / "wheels"
webui = temporary / "webui"
wheels.mkdir()
webui.mkdir()
_download_wheels(
packages=tuple(package_set["python"]),
destination=wheels,
python=args.python,
index_url=str(package_set["registries"]["python"]),
)
_download_webui(
packages=tuple(package_set["webui"]),
destination=webui,
npm=args.npm,
registry=str(package_set["registries"]["npm"]),
)
python_rows = _verify_wheels(tuple(package_set["python"]), wheels)
webui_rows = _verify_webui(tuple(package_set["webui"]), webui)
lock: dict[str, object] = {
"schema_version": "1",
"release_version": package_set["release_version"],
"package_set_sha256": package_set["package_set_sha256"],
"registries": package_set["registries"],
"python": python_rows,
"webui": webui_rows,
}
lock["lock_sha256"] = _canonical_sha256(lock)
shutil.copytree(wheels, wheelhouse, dirs_exist_ok=True)
shutil.copytree(webui, webui_packages, dirs_exist_ok=True)
args.lock_output.parent.mkdir(parents=True, exist_ok=True)
args.lock_output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.requirements_output is not None:
_write_requirements(args.requirements_output, python_rows)
return lock
def _load_package_set(path: Path) -> dict[str, object]:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict) or value.get("schema_version") != "1":
raise PackageArtifactError("package set has an unsupported shape")
expected_hash = value.get("package_set_sha256")
unsigned = dict(value)
unsigned.pop("package_set_sha256", None)
if expected_hash != _canonical_sha256(unsigned):
raise PackageArtifactError("package set hash does not match its contents")
registries = value.get("registries")
if not isinstance(registries, dict) or set(registries) != {"python", "npm"}:
raise PackageArtifactError("package set registries are invalid")
for registry in registries.values():
parsed = urlsplit(str(registry))
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
raise PackageArtifactError("package registries must use credential-free HTTPS URLs")
for group in ("python", "webui"):
packages = value.get(group)
if not isinstance(packages, list) or not packages:
raise PackageArtifactError(f"package set {group} entries are missing")
_validate_package_entries(group, packages)
return value
def _validate_package_entries(group: str, packages: list[object]) -> None:
names: set[str] = set()
for raw in packages:
if not isinstance(raw, dict):
raise PackageArtifactError(f"package set {group} entry is malformed")
name = raw.get("name")
version = raw.get("version")
repository = raw.get("repository")
tag = raw.get("tag")
commit = raw.get("commit")
name_valid = (
isinstance(name, str)
and (NAME.fullmatch(name) if group == "python" else WEBUI_NAME.fullmatch(name))
)
if (
not name_valid
or name in names
or not isinstance(version, str)
or VERSION.fullmatch(version) is None
or not isinstance(repository, str)
or NAME.fullmatch(repository) is None
or tag != f"v{version}"
or not isinstance(commit, str)
or COMMIT.fullmatch(commit) is None
):
raise PackageArtifactError(f"package set {group} entry has an invalid identity")
names.add(name)
if group == "python":
extras = raw.get("extras")
if not isinstance(extras, list) or any(
not isinstance(item, str)
or re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None
for item in extras
):
raise PackageArtifactError("package set Python extras are invalid")
def _download_wheels(
*, packages: tuple[dict[str, object], ...], destination: Path, python: str, index_url: str
) -> None:
requirements = [_python_requirement(item) for item in packages]
environment = dict(os.environ)
environment["PIP_INDEX_URL"] = _authenticated_url(index_url)
environment["PIP_EXTRA_INDEX_URL"] = ""
environment["PIP_CONFIG_FILE"] = os.devnull
environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
subprocess.run(
[python, "-m", "pip", "download", "--no-deps", "--only-binary=:all:", "--dest", str(destination), *requirements],
check=True,
env=environment,
)
def _download_webui(
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
) -> None:
environment = dict(os.environ)
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
if token:
parsed = urlsplit(registry)
auth_path = f"//{parsed.netloc}{parsed.path}:_authToken={token}\n"
npmrc = tempfile.NamedTemporaryFile(prefix="govoplan-npmrc-", delete=False)
npmrc.write(f"@govoplan:registry={registry}\n{auth_path}".encode("utf-8"))
npmrc.close()
os.chmod(npmrc.name, 0o600)
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
try:
for item in packages:
subprocess.run(
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
check=True,
env=environment,
)
finally:
if npmrc is not None:
Path(npmrc.name).unlink(missing_ok=True)
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
expected = {_normalize(str(item["name"])): item for item in packages}
rows: list[dict[str, object]] = []
seen: set[str] = set()
for path in sorted(root.glob("*.whl")):
identity = _wheel_identity(path)
name = str(identity["name"])
package = expected.get(name)
if package is None or identity["version"] != package["version"] or name in seen:
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
seen.add(name)
rows.append(_artifact_row(path, package))
if seen != set(expected):
raise PackageArtifactError("registry did not return every selected Python wheel")
return sorted(rows, key=lambda item: str(item["name"]))
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
expected = {str(item["name"]): item for item in packages}
rows: list[dict[str, object]] = []
seen: set[str] = set()
for path in sorted(root.glob("*.tgz")):
identity = _npm_identity(path)
name = str(identity["name"])
package = expected.get(name)
if package is None or identity["version"] != package["version"] or name in seen:
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
seen.add(name)
row = _artifact_row(path, package)
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
rows.append(row)
if seen != set(expected):
raise PackageArtifactError("registry did not return every selected WebUI package")
return sorted(rows, key=lambda item: str(item["name"]))
def _wheel_identity(path: Path) -> dict[str, str]:
_bounded(path)
with zipfile.ZipFile(path) as archive:
metadata = [
item for item in archive.infolist()
if PurePosixPath(item.filename).name == "METADATA"
and PurePosixPath(item.filename).parent.name.endswith(".dist-info")
]
if len(metadata) != 1 or metadata[0].file_size > 1024 * 1024:
raise PackageArtifactError(f"wheel metadata is invalid: {path.name}")
parsed = BytesParser(policy=compat32).parsebytes(archive.read(metadata[0]))
name = _normalize(str(parsed.get("Name") or ""))
version = str(parsed.get("Version") or "")
if NAME.fullmatch(name) is None or VERSION.fullmatch(version) is None:
raise PackageArtifactError(f"wheel identity is invalid: {path.name}")
return {"name": name, "version": version}
def _npm_identity(path: Path) -> dict[str, str]:
_bounded(path)
with tarfile.open(path, mode="r:gz") as archive:
try:
member = archive.getmember("package/package.json")
except KeyError as exc:
raise PackageArtifactError(f"npm package metadata is missing: {path.name}") from exc
if not member.isfile() or member.size > 1024 * 1024:
raise PackageArtifactError(f"npm package metadata is invalid: {path.name}")
extracted = archive.extractfile(member)
if extracted is None:
raise PackageArtifactError(f"npm package metadata cannot be read: {path.name}")
value = json.load(extracted)
name = value.get("name")
version = value.get("version")
if not isinstance(name, str) or not name.startswith("@govoplan/") or not isinstance(version, str) or VERSION.fullmatch(version) is None:
raise PackageArtifactError(f"npm package identity is invalid: {path.name}")
return {"name": name, "version": version}
def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
row = {
"name": package["name"],
"version": package["version"],
"repository": package["repository"],
"tag": package["tag"],
"commit": package["commit"],
"filename": path.name,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"size": path.stat().st_size,
}
if "extras" in package:
row["extras"] = package["extras"]
return row
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
for row in rows:
selected_extras = row.get("extras") or []
extras = (
f"[{','.join(str(value) for value in selected_extras)}]"
if selected_extras
else ""
)
lines.append(
f"{row['name']}{extras}=={row['version']} "
f"--hash=sha256:{row['sha256']}"
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _python_requirement(item: dict[str, object]) -> str:
extras = item.get("extras") or []
suffix = f"[{','.join(str(value) for value in extras)}]" if extras else ""
return f"{item['name']}{suffix}=={item['version']}"
def _authenticated_url(url: str) -> str:
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
username = os.environ.get("GOVOPLAN_PACKAGE_USERNAME", "")
if not token:
return url
if not username:
raise PackageArtifactError("GOVOPLAN_PACKAGE_USERNAME is required with a package token")
parsed = urlsplit(url)
return urlunsplit(
(
parsed.scheme,
f"{quote(username, safe='')}:{quote(token, safe='')}@{parsed.netloc}",
parsed.path,
parsed.query,
"",
)
)
def _require_empty_destination(path: Path) -> None:
if path.exists() and (not path.is_dir() or any(path.iterdir())):
raise PackageArtifactError(f"output directory must be absent or empty: {path}")
if path.is_symlink():
raise PackageArtifactError(f"output directory must not be a symlink: {path}")
def _bounded(path: Path) -> None:
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_ARTIFACT_BYTES:
raise PackageArtifactError(f"package artifact is invalid or too large: {path.name}")
def _normalize(value: str) -> str:
return re.sub(r"[-_.]+", "-", value.strip().lower())
def _canonical_sha256(value: object) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
def main() -> int:
args = build_parser().parse_args()
try:
lock = resolve(args)
except (PackageArtifactError, OSError, ValueError, subprocess.CalledProcessError, zipfile.BadZipFile, tarfile.TarError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
print(f"Resolved {len(lock['python'])} Python and {len(lock['webui'])} WebUI packages.")
print(f"Package artifact lock written to {args.lock_output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Install or verify the canonical package-release workflow in module repos."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
META_ROOT = Path(__file__).resolve().parents[2]
TEMPLATE = META_ROOT / "tools" / "repo" / "templates" / "module-package-release.yml"
DESTINATION = Path(".gitea/workflows/module-package-release.yml")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--check", action="store_true")
mode.add_argument("--write", action="store_true")
parser.add_argument(
"--parent",
type=Path,
default=META_ROOT.parent,
help="Parent directory containing the repositories.",
)
return parser
def package_repositories(parent: Path) -> tuple[Path, ...]:
inventory = json.loads((META_ROOT / "repositories.json").read_text(encoding="utf-8"))
repositories: list[Path] = []
for item in inventory["repositories"]:
name = str(item["name"])
if not name.startswith("govoplan-"):
continue
repository = parent / str(item["path"])
if (repository / "pyproject.toml").is_file():
repositories.append(repository)
return tuple(sorted(repositories))
def synchronize(*, parent: Path, write: bool) -> tuple[str, ...]:
expected = TEMPLATE.read_bytes()
mismatches: list[str] = []
for repository in package_repositories(parent):
destination = repository / DESTINATION
current = destination.read_bytes() if destination.is_file() else None
if current == expected:
continue
mismatches.append(repository.name)
if write:
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.with_suffix(destination.suffix + ".tmp")
temporary.write_bytes(expected)
temporary.chmod(0o644)
temporary.replace(destination)
return tuple(mismatches)
def main() -> int:
args = build_parser().parse_args()
mismatches = synchronize(parent=args.parent.expanduser().resolve(), write=args.write)
if args.write:
print(f"Installed package-release workflow in {len(mismatches)} repositories.")
return 0
if mismatches:
print("Package-release workflow is missing or stale in:", file=sys.stderr)
for repository in mismatches:
print(f"- {repository}", file=sys.stderr)
return 1
print("Package-release workflows are synchronized.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,209 @@
name: Module Package Release
on:
push:
tags:
- "v*"
workflow_dispatch:
inputs:
release_tag:
description: Existing protected version tag to publish
required: true
type: string
jobs:
publish-packages:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
fetch-depth: 0
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Select and validate protected release tag
shell: bash
env:
REQUESTED_TAG: ${{ inputs.release_tag }}
TRIGGER_TAG: ${{ gitea.ref_name }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
run: |
set -euo pipefail
tag="${REQUESTED_TAG:-$TRIGGER_TAG}"
case "$tag" in
v[0-9]*.[0-9]*.[0-9]*) ;;
*) echo "Release tag must start with a SemVer-shaped vX.Y.Z value" >&2; exit 1 ;;
esac
git fetch --force origin "refs/tags/$tag:refs/tags/$tag" refs/heads/main:refs/remotes/origin/main
tag_commit="$(git rev-list -n 1 "$tag")"
git merge-base --is-ancestor "$tag_commit" refs/remotes/origin/main || {
echo "Release tag is not contained in main" >&2
exit 1
}
python - "$tag" <<'PY'
import fnmatch
import json
import os
import sys
import urllib.request
tag = sys.argv[1]
repository = os.environ["GITEA_REPOSITORY"]
request = urllib.request.Request(
f"{os.environ['GITEA_API_URL']}/repos/{repository}/tag_protections",
headers={"Authorization": f"token {os.environ['GITEA_TOKEN']}"},
)
with urllib.request.urlopen(request, timeout=30) as response:
protections = json.load(response)
if not any(fnmatch.fnmatchcase(tag, item.get("name_pattern", "")) for item in protections):
raise SystemExit(f"Release tag {tag!r} is not covered by repository tag protection")
PY
git checkout --detach "$tag"
printf 'RELEASE_TAG=%s\n' "$tag" >> "$GITEA_ENV"
printf 'SOURCE_DATE_EPOCH=%s\n' "$(git show -s --format=%ct HEAD)" >> "$GITEA_ENV"
- name: Validate package versions
run: |
python - <<'PY'
import json
from pathlib import Path
import os
import re
import tomllib
tag = os.environ["RELEASE_TAG"]
expected = tag.removeprefix("v")
project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
if project.get("version") != expected:
raise SystemExit(f"pyproject version {project.get('version')!r} does not match {tag}")
if re.fullmatch(r"govoplan-[a-z0-9-]+", str(project.get("name", ""))) is None:
raise SystemExit("Python distribution name must use the govoplan-* namespace")
webui = Path("webui/package.json")
if webui.is_file():
package = json.loads(webui.read_text(encoding="utf-8"))
if package.get("version") != expected:
raise SystemExit(f"WebUI version {package.get('version')!r} does not match {tag}")
if re.fullmatch(r"@govoplan/[a-z0-9-]+-webui", str(package.get("name", ""))) is None:
raise SystemExit("WebUI package name must use the @govoplan/*-webui namespace")
release = Path("webui/package.release.json")
if release.is_file():
release_package = json.loads(release.read_text(encoding="utf-8"))
if (
release_package.get("name") != package.get("name")
or release_package.get("version") != expected
):
raise SystemExit("WebUI release package identity does not match package.json and the release tag")
PY
- name: Build immutable package artifacts
shell: bash
run: |
set -euo pipefail
python -m pip install --disable-pip-version-check build==1.5.0 twine==7.0.0
rm -rf dist .package-webui
python -m build --wheel --outdir dist
python -m twine check dist/*.whl
if [[ -f webui/package.json ]]; then
mkdir .package-webui
cp -a webui/. .package-webui/
rm -rf .package-webui/node_modules .package-webui/dist
if [[ -f .package-webui/package.release.json ]]; then
cp .package-webui/package.release.json .package-webui/package.json
fi
node <<'NODE'
const fs = require("node:fs");
const path = ".package-webui/package.json";
const packageJson = JSON.parse(fs.readFileSync(path, "utf8"));
const groups = ["dependencies", "optionalDependencies", "peerDependencies"];
for (const group of groups) {
for (const [name, specifier] of Object.entries(packageJson[group] || {})) {
if (!name.startsWith("@govoplan/")) continue;
if (typeof specifier !== "string") {
throw new Error(`${group}.${name} must use a string version`);
}
const packageSlug = name.slice("@govoplan/".length);
if (!packageSlug.endsWith("-webui")) {
throw new Error(`${group}.${name} is outside the WebUI package namespace`);
}
const repository = `govoplan-${packageSlug.slice(0, -"-webui".length)}`;
const escapedRepository = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const gitTag = specifier.match(
new RegExp(
`^git\\+(?:ssh://git@|https://)git\\.add-ideas\\.de/GovOPlaN/${escapedRepository}\\.git#v([0-9]+\\.[0-9]+\\.[0-9]+)$`,
),
);
if (gitTag) {
packageJson[group][name] = gitTag[1];
continue;
}
if (specifier.startsWith("file:") || specifier.startsWith("git+")) {
throw new Error(
`${group}.${name} must resolve to an exact registry version for publication`,
);
}
}
}
delete packageJson.private;
fs.writeFileSync(path, `${JSON.stringify(packageJson, null, 2)}\n`);
NODE
npm pkg delete private --prefix .package-webui
(cd .package-webui && npm pack --ignore-scripts --pack-destination ../dist)
fi
python - <<'PY'
import hashlib
import json
from pathlib import Path
import os
import subprocess
artifacts = []
for path in sorted(Path("dist").iterdir()):
if path.suffix not in {".whl", ".tgz"}:
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
artifacts.append({"filename": path.name, "sha256": digest, "size": path.stat().st_size})
payload = {
"schema_version": "1",
"repository": os.environ["GITEA_REPOSITORY"],
"tag": os.environ["RELEASE_TAG"],
"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
"artifacts": artifacts,
}
Path("dist/package-artifacts.json").write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
PY
- name: Retain package hash evidence
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
with:
name: module-packages-${{ gitea.ref_name }}
path: dist/package-artifacts.json
- name: Publish wheel and WebUI package
shell: bash
env:
PACKAGE_USERNAME: ${{ secrets.GOVOPLAN_PACKAGE_USERNAME }}
PACKAGE_TOKEN: ${{ secrets.GOVOPLAN_PACKAGE_TOKEN }}
run: |
set -euo pipefail
test -n "$PACKAGE_USERNAME"
test -n "$PACKAGE_TOKEN"
TWINE_USERNAME="$PACKAGE_USERNAME" TWINE_PASSWORD="$PACKAGE_TOKEN" \
python -m twine upload --non-interactive \
--repository-url https://git.add-ideas.de/api/packages/GovOPlaN/pypi \
dist/*.whl
shopt -s nullglob
webui_packages=(dist/*.tgz)
if (( ${#webui_packages[@]} )); then
npmrc="$(mktemp)"
trap 'rm -f "$npmrc"' EXIT
chmod 600 "$npmrc"
printf '%s\n' \
'@govoplan:registry=https://git.add-ideas.de/api/packages/GovOPlaN/npm/' \
"//git.add-ideas.de/api/packages/GovOPlaN/npm/:_authToken=$PACKAGE_TOKEN" \
> "$npmrc"
NPM_CONFIG_USERCONFIG="$npmrc" npm publish "${webui_packages[0]}" \
--ignore-scripts --access public \
--registry https://git.add-ideas.de/api/packages/GovOPlaN/npm/
fi