Add registry-backed module package releases
Dependency Audit / dependency-audit (push) Failing after 1m37s
Deployment Installer / deployment-installer (push) Successful in 6s
Security Audit / security-audit (push) Successful in 10m21s

This commit is contained in:
2026-08-04 04:14:28 +02:00
parent 1a68565ba0
commit f7590a7b8b
25 changed files with 1887 additions and 5 deletions
@@ -26,6 +26,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--web-metadata", type=Path, required=True)
parser.add_argument("--deployer", type=Path, required=True)
parser.add_argument("--deployer-url", required=True)
parser.add_argument("--package-lock", type=Path, required=True)
parser.add_argument("--artifact-base-url", required=True)
parser.add_argument("--source-commit", required=True)
parser.add_argument("--version", required=True)
@@ -45,6 +46,11 @@ def build_parser() -> argparse.ArgumentParser:
def finalize(args: argparse.Namespace) -> dict[str, Any]:
composition = _json_object(args.composition)
_validate_package_lock(
_json_object(args.package_lock),
version=args.version,
composition=composition,
)
api = _image_metadata(_json_object(args.api_metadata), "api")
web = _image_metadata(_json_object(args.web_metadata), "web")
dependencies = dict(_dependency(value) for value in args.dependency)
@@ -99,6 +105,10 @@ def finalize(args: argparse.Namespace) -> dict[str, Any]:
"url": args.deployer_url,
"sha256": _sha256_file(args.deployer),
},
"package_lock": {
"url": f"{artifact_base}/package-artifacts.lock.json",
"sha256": _sha256_file(args.package_lock),
},
"images": {
"api": {
**api,
@@ -247,6 +257,57 @@ def _json_object(path: Path) -> dict[str, Any]:
return value
def _validate_package_lock(
lock: dict[str, Any],
*,
version: str,
composition: dict[str, Any],
) -> None:
if lock.get("schema_version") != "1" or lock.get("release_version") != version:
raise ValueError("package lock schema or release version does not match")
unsigned = dict(lock)
expected_hash = unsigned.pop("lock_sha256", None)
actual_hash = hashlib.sha256(
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if expected_hash != actual_hash:
raise ValueError("package lock hash does not match its contents")
rows = lock.get("python")
if not isinstance(rows, list):
raise ValueError("package lock Python artifacts are missing")
locked = _package_identities(rows, name_key="name")
composed = _package_identities(
composition["python"]["packages"],
name_key="package",
)
if locked != composed:
raise ValueError("package lock does not match the runtime wheel composition")
def _package_identities(
rows: list[object],
*,
name_key: str,
) -> dict[str, tuple[str, str]]:
identities: dict[str, tuple[str, str]] = {}
for row in rows:
if not isinstance(row, dict):
raise ValueError("package artifact identity is malformed")
name = row.get(name_key)
version = row.get("version")
digest = row.get("sha256")
if (
not isinstance(name, str)
or not isinstance(version, str)
or not isinstance(digest, str)
or SHA256.fullmatch(digest) is None
or name in identities
):
raise ValueError("package artifact identity is malformed or duplicated")
identities[name] = (version, digest)
return identities
def _https_url(value: str, label: str) -> str:
parsed = urlsplit(value)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Generate the optional GovOPlaN developer convenience meta-package."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import re
import tomllib
META_ROOT = Path(__file__).resolve().parents[2]
DIRECT = re.compile(r"^(govoplan-[a-z0-9-]+)(?:\[([^]]+)\])?\s+@\s+.*@v([A-Za-z0-9._+!-]+)$")
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[([^]]+)\])?$")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument(
"--requirements",
type=Path,
default=META_ROOT / "requirements-release.txt",
)
parser.add_argument(
"--output",
type=Path,
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
)
parser.add_argument("--check", action="store_true")
return parser
def render(*, workspace: Path, requirements: Path) -> str:
core = tomllib.loads((workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8"))["project"]
version = str(core["version"])
base: list[str] = []
for raw in requirements.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
local = LOCAL_CORE.fullmatch(line)
if local:
extra = f"[{local.group(1)}]" if local.group(1) else ""
base.append(f"govoplan-core{extra}=={version}")
continue
match = DIRECT.fullmatch(line)
if match is None:
raise ValueError(f"unsupported release requirement: {line!r}")
extra = f"[{match.group(2)}]" if match.group(2) else ""
base.append(f"{match.group(1)}{extra}=={match.group(3)}")
base_names = {_requirement_name(item) for item in base}
full: list[str] = []
for project_path in sorted(workspace.glob("govoplan-*/pyproject.toml")):
project = tomllib.loads(project_path.read_text(encoding="utf-8"))["project"]
name = str(project.get("name") or "")
package_version = str(project.get("version") or "")
if name.startswith("govoplan-") and name not in base_names:
full.append(f"{name}=={package_version}")
return "\n".join(
[
"[build-system]",
'requires = ["setuptools>=69", "wheel"]',
'build-backend = "setuptools.build_meta"',
"",
"[project]",
'name = "govoplan"',
f'version = {json.dumps(version)}',
'description = "Developer convenience package for a versioned GovOPlaN composition"',
'readme = "README.md"',
'requires-python = ">=3.12"',
'license = { text = "AGPL-3.0-or-later" }',
"dependencies = [",
*[f" {json.dumps(item)}," for item in base],
"]",
"",
"[project.optional-dependencies]",
"full = [",
*[f" {json.dumps(item)}," for item in full],
"]",
"",
"[project.urls]",
'Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"',
'Documentation = "https://govoplan.add-ideas.de"',
"",
"[tool.setuptools.packages.find]",
'where = ["src"]',
"",
]
)
def _requirement_name(value: str) -> str:
return value.split("[", 1)[0].split("==", 1)[0]
def main() -> int:
args = build_parser().parse_args()
try:
expected = render(
workspace=args.workspace.expanduser().resolve(),
requirements=args.requirements.expanduser().resolve(),
)
except (OSError, KeyError, ValueError, tomllib.TOMLDecodeError) as exc:
print(f"error: {exc}")
return 1
output = args.output.expanduser()
current = output.read_text(encoding="utf-8") if output.is_file() else None
if args.check:
if current != expected:
print(f"error: developer meta-package is stale: {output}")
return 1
print("Developer meta-package is synchronized.")
return 0
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(expected, encoding="utf-8")
print(f"Developer meta-package written to {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Generate the exact registry package set for a GovOPlaN runtime release."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import subprocess
import tomllib
META_ROOT = Path(__file__).resolve().parents[2]
NAME = re.compile(r"^govoplan-[a-z0-9-]+$")
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
GIT_REQUIREMENT = re.compile(
r"^(?P<package>govoplan-[a-z0-9-]+)(?:\[(?P<extras>[^]]+)\])?\s+@\s+"
r"(?P<url>git\+[^\s]+/GovOPlaN/(?P<repo>govoplan-[a-z0-9-]+)\.git@v"
r"(?P<version>[A-Za-z0-9._+!-]+))$"
)
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
class PackageSetError(ValueError):
"""Release source references cannot form an immutable package set."""
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--version",
help="Core/meta release version. Defaults to the workspace Core version.",
)
parser.add_argument(
"--requirements",
type=Path,
default=META_ROOT / "requirements-release.txt",
)
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
parser.add_argument("--output", type=Path, required=True)
return parser
def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
values: list[dict[str, object]] = []
for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
local = LOCAL_CORE.fullmatch(line)
if local:
values.append(
{
"name": "govoplan-core",
"version": core_version.removeprefix("v"),
"repository": "govoplan-core",
"extras": _extras(local.group("extras")),
}
)
continue
match = GIT_REQUIREMENT.fullmatch(line)
if match is None:
raise PackageSetError(
f"unsupported release requirement at {path}:{line_number}: {line!r}"
)
values.append(
{
"name": match.group("package"),
"version": match.group("version"),
"repository": match.group("repo"),
"extras": _extras(match.group("extras")),
}
)
names = [str(item["name"]) for item in values]
if not values or names.count("govoplan-core") != 1 or len(names) != len(set(names)):
raise PackageSetError("release requirements must contain one Core and unique packages")
return tuple(values)
def generate_package_set(
*,
core_version: str,
requirements: Path,
workspace: Path,
) -> dict[str, object]:
core_version = core_version.removeprefix("v")
if VERSION.fullmatch(core_version) is None:
raise PackageSetError("release version is invalid")
python_packages: list[dict[str, object]] = []
webui_packages: list[dict[str, object]] = []
seen_webui: set[str] = set()
for requirement in parse_release_requirements(requirements, core_version=core_version):
repository = workspace / str(requirement["repository"])
tag = f"v{requirement['version']}"
if not (repository / ".git").is_dir():
raise PackageSetError(f"release repository is missing: {repository}")
commit = _git(repository, "rev-list", "-n", "1", tag)
if not commit:
raise PackageSetError(f"release tag is missing: {repository.name}@{tag}")
project = tomllib.loads(_git(repository, "show", f"{tag}:pyproject.toml"))["project"]
if project.get("name") != requirement["name"] or project.get("version") != requirement["version"]:
raise PackageSetError(f"tag metadata does not match {repository.name}@{tag}")
entry = {
**requirement,
"tag": tag,
"commit": commit,
}
python_packages.append(entry)
try:
webui_raw = _git(repository, "show", f"{tag}:webui/package.json")
except subprocess.CalledProcessError:
continue
webui = json.loads(webui_raw)
webui_name = webui.get("name")
if (
not isinstance(webui_name, str)
or not webui_name.startswith("@govoplan/")
or webui.get("version") != requirement["version"]
or webui_name in seen_webui
):
raise PackageSetError(f"WebUI tag metadata does not match {repository.name}@{tag}")
seen_webui.add(webui_name)
webui_packages.append(
{
"name": webui_name,
"version": requirement["version"],
"repository": requirement["repository"],
"tag": tag,
"commit": commit,
}
)
payload: dict[str, object] = {
"schema_version": "1",
"release_version": core_version,
"registries": {
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
},
"python": python_packages,
"webui": webui_packages,
}
payload["package_set_sha256"] = _canonical_sha256(payload)
return payload
def _extras(value: str | None) -> list[str]:
if not value:
return []
extras = sorted({item.strip() for item in value.split(",") if item.strip()})
if any(re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None for item in extras):
raise PackageSetError("release requirement contains an invalid extra")
return extras
def _git(repository: Path, *arguments: str) -> str:
return subprocess.check_output(
["git", "-C", str(repository), *arguments],
text=True,
stderr=subprocess.DEVNULL,
).strip()
def _canonical_sha256(value: object) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def main() -> int:
args = build_parser().parse_args()
try:
workspace = args.workspace.expanduser().resolve()
version = args.version
if not version:
core = tomllib.loads(
(workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8")
)
version = str(core["project"]["version"])
payload = generate_package_set(
core_version=version,
requirements=args.requirements.expanduser().resolve(),
workspace=workspace,
)
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
print(f"error: {exc}")
return 1
output = args.output.expanduser()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"Release package set written to {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -8,6 +8,9 @@ WEBUI_DIR="${1:-$CORE_ROOT/webui}"
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
export GOVOPLAN_DEPS
PACKAGE_LOCK="${GOVOPLAN_WEBUI_PACKAGE_LOCK:-}"
PACKAGE_DIR="${GOVOPLAN_WEBUI_PACKAGE_DIR:-}"
PYTHON_BIN="${PYTHON:-python3}"
trap 'rm -rf "$WORK_ROOT"' EXIT
@@ -55,9 +58,69 @@ rm -f package-lock.json
npm cache clean --force
retry npm install --prefer-online
if [[ -n "$PACKAGE_LOCK" || -n "$PACKAGE_DIR" ]]; then
[[ -n "$PACKAGE_LOCK" && -n "$PACKAGE_DIR" ]] || {
echo "GOVOPLAN_WEBUI_PACKAGE_LOCK and GOVOPLAN_WEBUI_PACKAGE_DIR must be set together" >&2
exit 1
}
"$PYTHON_BIN" - "$PACKAGE_LOCK" "$PACKAGE_DIR" "$GOVOPLAN_DEPS" <<'PY'
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import sys
lock_path = Path(sys.argv[1]).resolve()
package_dir = Path(sys.argv[2]).resolve()
output = Path(sys.argv[3])
lock = json.loads(lock_path.read_text(encoding="utf-8"))
if lock.get("schema_version") != "1" or not isinstance(lock.get("webui"), list):
raise SystemExit("WebUI package lock is malformed")
unsigned = dict(lock)
expected_lock_hash = unsigned.pop("lock_sha256", None)
actual_lock_hash = hashlib.sha256(
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if expected_lock_hash != actual_lock_hash:
raise SystemExit("WebUI package lock hash does not match its contents")
rows = {}
for item in lock["webui"]:
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
raise SystemExit("WebUI package lock contains a malformed artifact")
if item["name"] in rows:
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
rows[item["name"]] = item
requested = []
for line in output.read_text(encoding="utf-8").splitlines():
if not line:
continue
name, _source_ref = line.split("\t", 1)
row = rows.get(name)
if not isinstance(row, dict):
raise SystemExit(f"WebUI package lock has no artifact for {name}")
filename = row.get("filename")
if not isinstance(filename, str) or Path(filename).name != filename:
raise SystemExit(f"WebUI package lock has an invalid filename for {name}")
artifact = package_dir / filename
if artifact.is_symlink() or not artifact.is_file():
raise SystemExit(f"WebUI package artifact is missing for {name}")
encoded = artifact.read_bytes()
if len(encoded) != row.get("size") or hashlib.sha256(encoded).hexdigest() != row.get("sha256"):
raise SystemExit(f"WebUI package artifact hash does not match for {name}")
requested.append(f"{name}\tfile:{artifact}")
output.write_text("\n".join(requested) + "\n", encoding="utf-8")
PY
fi
module_paths=()
while IFS=$'\t' read -r package_name spec; do
[[ -n "${package_name:-}" ]] || continue
if [[ "$spec" == file:* ]]; then
echo "Installing $package_name from verified package artifact"
module_paths+=("$spec")
continue
fi
git_url="${spec%%#*}"
git_ref="${spec#*#}"
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
+9
View File
@@ -456,6 +456,13 @@ path.write_text(updated)
PYCODE
}
update_developer_meta_package() {
"$PYTHON" "$META_ROOT/tools/release/generate-developer-meta-package.py" \
--workspace "$PARENT" \
--requirements "$META_ROOT/requirements-release.txt" \
--output "$META_ROOT/packages/govoplan-meta/pyproject.toml"
}
update_version_files() {
local repo="$1"
local version="$2"
@@ -875,8 +882,10 @@ done
if [[ "$DRY_RUN" -eq 1 ]]; then
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
echo "Would synchronize packages/govoplan-meta/pyproject.toml"
else
update_release_requirements "$TARGET_VERSION"
update_developer_meta_package
fi
refresh_development_webui_lock
+361
View File
@@ -0,0 +1,361 @@
#!/usr/bin/env python3
"""Download, verify, and lock exact GovOPlaN registry package artifacts."""
from __future__ import annotations
import argparse
import base64
from email.parser import BytesParser
from email.policy import compat32
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
from urllib.parse import quote, urlsplit, urlunsplit
import zipfile
NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
WEBUI_NAME = re.compile(r"^@govoplan/[a-z0-9]+(?:-[a-z0-9]+)*-webui$")
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
COMMIT = re.compile(r"^[0-9a-f]{40}$")
MAX_ARTIFACT_BYTES = 512 * 1024 * 1024
class PackageArtifactError(ValueError):
"""Registry artifacts do not match the selected package set."""
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package-set", type=Path, required=True)
parser.add_argument("--wheelhouse", type=Path, required=True)
parser.add_argument("--webui-packages", type=Path, required=True)
parser.add_argument("--lock-output", type=Path, required=True)
parser.add_argument("--requirements-output", type=Path)
parser.add_argument("--python", default=sys.executable)
parser.add_argument("--npm", default="npm")
return parser
def resolve(args: argparse.Namespace) -> dict[str, object]:
package_set = _load_package_set(args.package_set)
wheelhouse = args.wheelhouse.expanduser().resolve()
webui_packages = args.webui_packages.expanduser().resolve()
_require_empty_destination(wheelhouse)
_require_empty_destination(webui_packages)
wheelhouse.parent.mkdir(parents=True, exist_ok=True)
webui_packages.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="govoplan-package-resolution-") as value:
temporary = Path(value)
wheels = temporary / "wheels"
webui = temporary / "webui"
wheels.mkdir()
webui.mkdir()
_download_wheels(
packages=tuple(package_set["python"]),
destination=wheels,
python=args.python,
index_url=str(package_set["registries"]["python"]),
)
_download_webui(
packages=tuple(package_set["webui"]),
destination=webui,
npm=args.npm,
registry=str(package_set["registries"]["npm"]),
)
python_rows = _verify_wheels(tuple(package_set["python"]), wheels)
webui_rows = _verify_webui(tuple(package_set["webui"]), webui)
lock: dict[str, object] = {
"schema_version": "1",
"release_version": package_set["release_version"],
"package_set_sha256": package_set["package_set_sha256"],
"registries": package_set["registries"],
"python": python_rows,
"webui": webui_rows,
}
lock["lock_sha256"] = _canonical_sha256(lock)
shutil.copytree(wheels, wheelhouse, dirs_exist_ok=True)
shutil.copytree(webui, webui_packages, dirs_exist_ok=True)
args.lock_output.parent.mkdir(parents=True, exist_ok=True)
args.lock_output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.requirements_output is not None:
_write_requirements(args.requirements_output, python_rows)
return lock
def _load_package_set(path: Path) -> dict[str, object]:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict) or value.get("schema_version") != "1":
raise PackageArtifactError("package set has an unsupported shape")
expected_hash = value.get("package_set_sha256")
unsigned = dict(value)
unsigned.pop("package_set_sha256", None)
if expected_hash != _canonical_sha256(unsigned):
raise PackageArtifactError("package set hash does not match its contents")
registries = value.get("registries")
if not isinstance(registries, dict) or set(registries) != {"python", "npm"}:
raise PackageArtifactError("package set registries are invalid")
for registry in registries.values():
parsed = urlsplit(str(registry))
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
raise PackageArtifactError("package registries must use credential-free HTTPS URLs")
for group in ("python", "webui"):
packages = value.get(group)
if not isinstance(packages, list) or not packages:
raise PackageArtifactError(f"package set {group} entries are missing")
_validate_package_entries(group, packages)
return value
def _validate_package_entries(group: str, packages: list[object]) -> None:
names: set[str] = set()
for raw in packages:
if not isinstance(raw, dict):
raise PackageArtifactError(f"package set {group} entry is malformed")
name = raw.get("name")
version = raw.get("version")
repository = raw.get("repository")
tag = raw.get("tag")
commit = raw.get("commit")
name_valid = (
isinstance(name, str)
and (NAME.fullmatch(name) if group == "python" else WEBUI_NAME.fullmatch(name))
)
if (
not name_valid
or name in names
or not isinstance(version, str)
or VERSION.fullmatch(version) is None
or not isinstance(repository, str)
or NAME.fullmatch(repository) is None
or tag != f"v{version}"
or not isinstance(commit, str)
or COMMIT.fullmatch(commit) is None
):
raise PackageArtifactError(f"package set {group} entry has an invalid identity")
names.add(name)
if group == "python":
extras = raw.get("extras")
if not isinstance(extras, list) or any(
not isinstance(item, str)
or re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None
for item in extras
):
raise PackageArtifactError("package set Python extras are invalid")
def _download_wheels(
*, packages: tuple[dict[str, object], ...], destination: Path, python: str, index_url: str
) -> None:
requirements = [_python_requirement(item) for item in packages]
environment = dict(os.environ)
environment["PIP_INDEX_URL"] = _authenticated_url(index_url)
environment["PIP_EXTRA_INDEX_URL"] = ""
environment["PIP_CONFIG_FILE"] = os.devnull
environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
subprocess.run(
[python, "-m", "pip", "download", "--no-deps", "--only-binary=:all:", "--dest", str(destination), *requirements],
check=True,
env=environment,
)
def _download_webui(
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
) -> None:
environment = dict(os.environ)
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
if token:
parsed = urlsplit(registry)
auth_path = f"//{parsed.netloc}{parsed.path}:_authToken={token}\n"
npmrc = tempfile.NamedTemporaryFile(prefix="govoplan-npmrc-", delete=False)
npmrc.write(f"@govoplan:registry={registry}\n{auth_path}".encode("utf-8"))
npmrc.close()
os.chmod(npmrc.name, 0o600)
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
try:
for item in packages:
subprocess.run(
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
check=True,
env=environment,
)
finally:
if npmrc is not None:
Path(npmrc.name).unlink(missing_ok=True)
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
expected = {_normalize(str(item["name"])): item for item in packages}
rows: list[dict[str, object]] = []
seen: set[str] = set()
for path in sorted(root.glob("*.whl")):
identity = _wheel_identity(path)
name = str(identity["name"])
package = expected.get(name)
if package is None or identity["version"] != package["version"] or name in seen:
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
seen.add(name)
rows.append(_artifact_row(path, package))
if seen != set(expected):
raise PackageArtifactError("registry did not return every selected Python wheel")
return sorted(rows, key=lambda item: str(item["name"]))
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
expected = {str(item["name"]): item for item in packages}
rows: list[dict[str, object]] = []
seen: set[str] = set()
for path in sorted(root.glob("*.tgz")):
identity = _npm_identity(path)
name = str(identity["name"])
package = expected.get(name)
if package is None or identity["version"] != package["version"] or name in seen:
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
seen.add(name)
row = _artifact_row(path, package)
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
rows.append(row)
if seen != set(expected):
raise PackageArtifactError("registry did not return every selected WebUI package")
return sorted(rows, key=lambda item: str(item["name"]))
def _wheel_identity(path: Path) -> dict[str, str]:
_bounded(path)
with zipfile.ZipFile(path) as archive:
metadata = [
item for item in archive.infolist()
if PurePosixPath(item.filename).name == "METADATA"
and PurePosixPath(item.filename).parent.name.endswith(".dist-info")
]
if len(metadata) != 1 or metadata[0].file_size > 1024 * 1024:
raise PackageArtifactError(f"wheel metadata is invalid: {path.name}")
parsed = BytesParser(policy=compat32).parsebytes(archive.read(metadata[0]))
name = _normalize(str(parsed.get("Name") or ""))
version = str(parsed.get("Version") or "")
if NAME.fullmatch(name) is None or VERSION.fullmatch(version) is None:
raise PackageArtifactError(f"wheel identity is invalid: {path.name}")
return {"name": name, "version": version}
def _npm_identity(path: Path) -> dict[str, str]:
_bounded(path)
with tarfile.open(path, mode="r:gz") as archive:
try:
member = archive.getmember("package/package.json")
except KeyError as exc:
raise PackageArtifactError(f"npm package metadata is missing: {path.name}") from exc
if not member.isfile() or member.size > 1024 * 1024:
raise PackageArtifactError(f"npm package metadata is invalid: {path.name}")
extracted = archive.extractfile(member)
if extracted is None:
raise PackageArtifactError(f"npm package metadata cannot be read: {path.name}")
value = json.load(extracted)
name = value.get("name")
version = value.get("version")
if not isinstance(name, str) or not name.startswith("@govoplan/") or not isinstance(version, str) or VERSION.fullmatch(version) is None:
raise PackageArtifactError(f"npm package identity is invalid: {path.name}")
return {"name": name, "version": version}
def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
row = {
"name": package["name"],
"version": package["version"],
"repository": package["repository"],
"tag": package["tag"],
"commit": package["commit"],
"filename": path.name,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"size": path.stat().st_size,
}
if "extras" in package:
row["extras"] = package["extras"]
return row
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
for row in rows:
selected_extras = row.get("extras") or []
extras = (
f"[{','.join(str(value) for value in selected_extras)}]"
if selected_extras
else ""
)
lines.append(
f"{row['name']}{extras}=={row['version']} "
f"--hash=sha256:{row['sha256']}"
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _python_requirement(item: dict[str, object]) -> str:
extras = item.get("extras") or []
suffix = f"[{','.join(str(value) for value in extras)}]" if extras else ""
return f"{item['name']}{suffix}=={item['version']}"
def _authenticated_url(url: str) -> str:
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
username = os.environ.get("GOVOPLAN_PACKAGE_USERNAME", "")
if not token:
return url
if not username:
raise PackageArtifactError("GOVOPLAN_PACKAGE_USERNAME is required with a package token")
parsed = urlsplit(url)
return urlunsplit(
(
parsed.scheme,
f"{quote(username, safe='')}:{quote(token, safe='')}@{parsed.netloc}",
parsed.path,
parsed.query,
"",
)
)
def _require_empty_destination(path: Path) -> None:
if path.exists() and (not path.is_dir() or any(path.iterdir())):
raise PackageArtifactError(f"output directory must be absent or empty: {path}")
if path.is_symlink():
raise PackageArtifactError(f"output directory must not be a symlink: {path}")
def _bounded(path: Path) -> None:
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_ARTIFACT_BYTES:
raise PackageArtifactError(f"package artifact is invalid or too large: {path.name}")
def _normalize(value: str) -> str:
return re.sub(r"[-_.]+", "-", value.strip().lower())
def _canonical_sha256(value: object) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
def main() -> int:
args = build_parser().parse_args()
try:
lock = resolve(args)
except (PackageArtifactError, OSError, ValueError, subprocess.CalledProcessError, zipfile.BadZipFile, tarfile.TarError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
print(f"Resolved {len(lock['python'])} Python and {len(lock['webui'])} WebUI packages.")
print(f"Package artifact lock written to {args.lock_output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())