Add registry-backed module package releases
This commit is contained in:
@@ -26,6 +26,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--web-metadata", type=Path, required=True)
|
||||
parser.add_argument("--deployer", type=Path, required=True)
|
||||
parser.add_argument("--deployer-url", required=True)
|
||||
parser.add_argument("--package-lock", type=Path, required=True)
|
||||
parser.add_argument("--artifact-base-url", required=True)
|
||||
parser.add_argument("--source-commit", required=True)
|
||||
parser.add_argument("--version", required=True)
|
||||
@@ -45,6 +46,11 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
def finalize(args: argparse.Namespace) -> dict[str, Any]:
|
||||
composition = _json_object(args.composition)
|
||||
_validate_package_lock(
|
||||
_json_object(args.package_lock),
|
||||
version=args.version,
|
||||
composition=composition,
|
||||
)
|
||||
api = _image_metadata(_json_object(args.api_metadata), "api")
|
||||
web = _image_metadata(_json_object(args.web_metadata), "web")
|
||||
dependencies = dict(_dependency(value) for value in args.dependency)
|
||||
@@ -99,6 +105,10 @@ def finalize(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"url": args.deployer_url,
|
||||
"sha256": _sha256_file(args.deployer),
|
||||
},
|
||||
"package_lock": {
|
||||
"url": f"{artifact_base}/package-artifacts.lock.json",
|
||||
"sha256": _sha256_file(args.package_lock),
|
||||
},
|
||||
"images": {
|
||||
"api": {
|
||||
**api,
|
||||
@@ -247,6 +257,57 @@ def _json_object(path: Path) -> dict[str, Any]:
|
||||
return value
|
||||
|
||||
|
||||
def _validate_package_lock(
|
||||
lock: dict[str, Any],
|
||||
*,
|
||||
version: str,
|
||||
composition: dict[str, Any],
|
||||
) -> None:
|
||||
if lock.get("schema_version") != "1" or lock.get("release_version") != version:
|
||||
raise ValueError("package lock schema or release version does not match")
|
||||
unsigned = dict(lock)
|
||||
expected_hash = unsigned.pop("lock_sha256", None)
|
||||
actual_hash = hashlib.sha256(
|
||||
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
if expected_hash != actual_hash:
|
||||
raise ValueError("package lock hash does not match its contents")
|
||||
rows = lock.get("python")
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("package lock Python artifacts are missing")
|
||||
locked = _package_identities(rows, name_key="name")
|
||||
composed = _package_identities(
|
||||
composition["python"]["packages"],
|
||||
name_key="package",
|
||||
)
|
||||
if locked != composed:
|
||||
raise ValueError("package lock does not match the runtime wheel composition")
|
||||
|
||||
|
||||
def _package_identities(
|
||||
rows: list[object],
|
||||
*,
|
||||
name_key: str,
|
||||
) -> dict[str, tuple[str, str]]:
|
||||
identities: dict[str, tuple[str, str]] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise ValueError("package artifact identity is malformed")
|
||||
name = row.get(name_key)
|
||||
version = row.get("version")
|
||||
digest = row.get("sha256")
|
||||
if (
|
||||
not isinstance(name, str)
|
||||
or not isinstance(version, str)
|
||||
or not isinstance(digest, str)
|
||||
or SHA256.fullmatch(digest) is None
|
||||
or name in identities
|
||||
):
|
||||
raise ValueError("package artifact identity is malformed or duplicated")
|
||||
identities[name] = (version, digest)
|
||||
return identities
|
||||
|
||||
|
||||
def _https_url(value: str, label: str) -> str:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the optional GovOPlaN developer convenience meta-package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tomllib
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
DIRECT = re.compile(r"^(govoplan-[a-z0-9-]+)(?:\[([^]]+)\])?\s+@\s+.*@v([A-Za-z0-9._+!-]+)$")
|
||||
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[([^]]+)\])?$")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||
parser.add_argument(
|
||||
"--requirements",
|
||||
type=Path,
|
||||
default=META_ROOT / "requirements-release.txt",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=META_ROOT / "packages" / "govoplan-meta" / "pyproject.toml",
|
||||
)
|
||||
parser.add_argument("--check", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def render(*, workspace: Path, requirements: Path) -> str:
|
||||
core = tomllib.loads((workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
version = str(core["version"])
|
||||
base: list[str] = []
|
||||
for raw in requirements.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
local = LOCAL_CORE.fullmatch(line)
|
||||
if local:
|
||||
extra = f"[{local.group(1)}]" if local.group(1) else ""
|
||||
base.append(f"govoplan-core{extra}=={version}")
|
||||
continue
|
||||
match = DIRECT.fullmatch(line)
|
||||
if match is None:
|
||||
raise ValueError(f"unsupported release requirement: {line!r}")
|
||||
extra = f"[{match.group(2)}]" if match.group(2) else ""
|
||||
base.append(f"{match.group(1)}{extra}=={match.group(3)}")
|
||||
|
||||
base_names = {_requirement_name(item) for item in base}
|
||||
full: list[str] = []
|
||||
for project_path in sorted(workspace.glob("govoplan-*/pyproject.toml")):
|
||||
project = tomllib.loads(project_path.read_text(encoding="utf-8"))["project"]
|
||||
name = str(project.get("name") or "")
|
||||
package_version = str(project.get("version") or "")
|
||||
if name.startswith("govoplan-") and name not in base_names:
|
||||
full.append(f"{name}=={package_version}")
|
||||
|
||||
return "\n".join(
|
||||
[
|
||||
"[build-system]",
|
||||
'requires = ["setuptools>=69", "wheel"]',
|
||||
'build-backend = "setuptools.build_meta"',
|
||||
"",
|
||||
"[project]",
|
||||
'name = "govoplan"',
|
||||
f'version = {json.dumps(version)}',
|
||||
'description = "Developer convenience package for a versioned GovOPlaN composition"',
|
||||
'readme = "README.md"',
|
||||
'requires-python = ">=3.12"',
|
||||
'license = { text = "AGPL-3.0-or-later" }',
|
||||
"dependencies = [",
|
||||
*[f" {json.dumps(item)}," for item in base],
|
||||
"]",
|
||||
"",
|
||||
"[project.optional-dependencies]",
|
||||
"full = [",
|
||||
*[f" {json.dumps(item)}," for item in full],
|
||||
"]",
|
||||
"",
|
||||
"[project.urls]",
|
||||
'Repository = "https://git.add-ideas.de/GovOPlaN/govoplan"',
|
||||
'Documentation = "https://govoplan.add-ideas.de"',
|
||||
"",
|
||||
"[tool.setuptools.packages.find]",
|
||||
'where = ["src"]',
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _requirement_name(value: str) -> str:
|
||||
return value.split("[", 1)[0].split("==", 1)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
try:
|
||||
expected = render(
|
||||
workspace=args.workspace.expanduser().resolve(),
|
||||
requirements=args.requirements.expanduser().resolve(),
|
||||
)
|
||||
except (OSError, KeyError, ValueError, tomllib.TOMLDecodeError) as exc:
|
||||
print(f"error: {exc}")
|
||||
return 1
|
||||
output = args.output.expanduser()
|
||||
current = output.read_text(encoding="utf-8") if output.is_file() else None
|
||||
if args.check:
|
||||
if current != expected:
|
||||
print(f"error: developer meta-package is stale: {output}")
|
||||
return 1
|
||||
print("Developer meta-package is synchronized.")
|
||||
return 0
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(expected, encoding="utf-8")
|
||||
print(f"Developer meta-package written to {output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the exact registry package set for a GovOPlaN runtime release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tomllib
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
NAME = re.compile(r"^govoplan-[a-z0-9-]+$")
|
||||
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
|
||||
GIT_REQUIREMENT = re.compile(
|
||||
r"^(?P<package>govoplan-[a-z0-9-]+)(?:\[(?P<extras>[^]]+)\])?\s+@\s+"
|
||||
r"(?P<url>git\+[^\s]+/GovOPlaN/(?P<repo>govoplan-[a-z0-9-]+)\.git@v"
|
||||
r"(?P<version>[A-Za-z0-9._+!-]+))$"
|
||||
)
|
||||
LOCAL_CORE = re.compile(r"^(?:-e\s+)?\.\./govoplan-core(?:\[(?P<extras>[^]]+)\])?$")
|
||||
|
||||
|
||||
class PackageSetError(ValueError):
|
||||
"""Release source references cannot form an immutable package set."""
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
help="Core/meta release version. Defaults to the workspace Core version.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--requirements",
|
||||
type=Path,
|
||||
default=META_ROOT / "requirements-release.txt",
|
||||
)
|
||||
parser.add_argument("--workspace", type=Path, default=META_ROOT.parent)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_release_requirements(path: Path, *, core_version: str) -> tuple[dict[str, object], ...]:
|
||||
values: list[dict[str, object]] = []
|
||||
for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
local = LOCAL_CORE.fullmatch(line)
|
||||
if local:
|
||||
values.append(
|
||||
{
|
||||
"name": "govoplan-core",
|
||||
"version": core_version.removeprefix("v"),
|
||||
"repository": "govoplan-core",
|
||||
"extras": _extras(local.group("extras")),
|
||||
}
|
||||
)
|
||||
continue
|
||||
match = GIT_REQUIREMENT.fullmatch(line)
|
||||
if match is None:
|
||||
raise PackageSetError(
|
||||
f"unsupported release requirement at {path}:{line_number}: {line!r}"
|
||||
)
|
||||
values.append(
|
||||
{
|
||||
"name": match.group("package"),
|
||||
"version": match.group("version"),
|
||||
"repository": match.group("repo"),
|
||||
"extras": _extras(match.group("extras")),
|
||||
}
|
||||
)
|
||||
names = [str(item["name"]) for item in values]
|
||||
if not values or names.count("govoplan-core") != 1 or len(names) != len(set(names)):
|
||||
raise PackageSetError("release requirements must contain one Core and unique packages")
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def generate_package_set(
|
||||
*,
|
||||
core_version: str,
|
||||
requirements: Path,
|
||||
workspace: Path,
|
||||
) -> dict[str, object]:
|
||||
core_version = core_version.removeprefix("v")
|
||||
if VERSION.fullmatch(core_version) is None:
|
||||
raise PackageSetError("release version is invalid")
|
||||
python_packages: list[dict[str, object]] = []
|
||||
webui_packages: list[dict[str, object]] = []
|
||||
seen_webui: set[str] = set()
|
||||
for requirement in parse_release_requirements(requirements, core_version=core_version):
|
||||
repository = workspace / str(requirement["repository"])
|
||||
tag = f"v{requirement['version']}"
|
||||
if not (repository / ".git").is_dir():
|
||||
raise PackageSetError(f"release repository is missing: {repository}")
|
||||
commit = _git(repository, "rev-list", "-n", "1", tag)
|
||||
if not commit:
|
||||
raise PackageSetError(f"release tag is missing: {repository.name}@{tag}")
|
||||
project = tomllib.loads(_git(repository, "show", f"{tag}:pyproject.toml"))["project"]
|
||||
if project.get("name") != requirement["name"] or project.get("version") != requirement["version"]:
|
||||
raise PackageSetError(f"tag metadata does not match {repository.name}@{tag}")
|
||||
entry = {
|
||||
**requirement,
|
||||
"tag": tag,
|
||||
"commit": commit,
|
||||
}
|
||||
python_packages.append(entry)
|
||||
try:
|
||||
webui_raw = _git(repository, "show", f"{tag}:webui/package.json")
|
||||
except subprocess.CalledProcessError:
|
||||
continue
|
||||
webui = json.loads(webui_raw)
|
||||
webui_name = webui.get("name")
|
||||
if (
|
||||
not isinstance(webui_name, str)
|
||||
or not webui_name.startswith("@govoplan/")
|
||||
or webui.get("version") != requirement["version"]
|
||||
or webui_name in seen_webui
|
||||
):
|
||||
raise PackageSetError(f"WebUI tag metadata does not match {repository.name}@{tag}")
|
||||
seen_webui.add(webui_name)
|
||||
webui_packages.append(
|
||||
{
|
||||
"name": webui_name,
|
||||
"version": requirement["version"],
|
||||
"repository": requirement["repository"],
|
||||
"tag": tag,
|
||||
"commit": commit,
|
||||
}
|
||||
)
|
||||
payload: dict[str, object] = {
|
||||
"schema_version": "1",
|
||||
"release_version": core_version,
|
||||
"registries": {
|
||||
"python": "https://git.add-ideas.de/api/packages/GovOPlaN/pypi/simple",
|
||||
"npm": "https://git.add-ideas.de/api/packages/GovOPlaN/npm/",
|
||||
},
|
||||
"python": python_packages,
|
||||
"webui": webui_packages,
|
||||
}
|
||||
payload["package_set_sha256"] = _canonical_sha256(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _extras(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
extras = sorted({item.strip() for item in value.split(",") if item.strip()})
|
||||
if any(re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None for item in extras):
|
||||
raise PackageSetError("release requirement contains an invalid extra")
|
||||
return extras
|
||||
|
||||
|
||||
def _git(repository: Path, *arguments: str) -> str:
|
||||
return subprocess.check_output(
|
||||
["git", "-C", str(repository), *arguments],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
try:
|
||||
workspace = args.workspace.expanduser().resolve()
|
||||
version = args.version
|
||||
if not version:
|
||||
core = tomllib.loads(
|
||||
(workspace / "govoplan-core/pyproject.toml").read_text(encoding="utf-8")
|
||||
)
|
||||
version = str(core["project"]["version"])
|
||||
payload = generate_package_set(
|
||||
core_version=version,
|
||||
requirements=args.requirements.expanduser().resolve(),
|
||||
workspace=workspace,
|
||||
)
|
||||
except (PackageSetError, OSError, ValueError, subprocess.CalledProcessError) as exc:
|
||||
print(f"error: {exc}")
|
||||
return 1
|
||||
output = args.output.expanduser()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(f"Release package set written to {output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -8,6 +8,9 @@ WEBUI_DIR="${1:-$CORE_ROOT/webui}"
|
||||
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
|
||||
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
|
||||
export GOVOPLAN_DEPS
|
||||
PACKAGE_LOCK="${GOVOPLAN_WEBUI_PACKAGE_LOCK:-}"
|
||||
PACKAGE_DIR="${GOVOPLAN_WEBUI_PACKAGE_DIR:-}"
|
||||
PYTHON_BIN="${PYTHON:-python3}"
|
||||
|
||||
trap 'rm -rf "$WORK_ROOT"' EXIT
|
||||
|
||||
@@ -55,9 +58,69 @@ rm -f package-lock.json
|
||||
npm cache clean --force
|
||||
retry npm install --prefer-online
|
||||
|
||||
if [[ -n "$PACKAGE_LOCK" || -n "$PACKAGE_DIR" ]]; then
|
||||
[[ -n "$PACKAGE_LOCK" && -n "$PACKAGE_DIR" ]] || {
|
||||
echo "GOVOPLAN_WEBUI_PACKAGE_LOCK and GOVOPLAN_WEBUI_PACKAGE_DIR must be set together" >&2
|
||||
exit 1
|
||||
}
|
||||
"$PYTHON_BIN" - "$PACKAGE_LOCK" "$PACKAGE_DIR" "$GOVOPLAN_DEPS" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
lock_path = Path(sys.argv[1]).resolve()
|
||||
package_dir = Path(sys.argv[2]).resolve()
|
||||
output = Path(sys.argv[3])
|
||||
lock = json.loads(lock_path.read_text(encoding="utf-8"))
|
||||
if lock.get("schema_version") != "1" or not isinstance(lock.get("webui"), list):
|
||||
raise SystemExit("WebUI package lock is malformed")
|
||||
unsigned = dict(lock)
|
||||
expected_lock_hash = unsigned.pop("lock_sha256", None)
|
||||
actual_lock_hash = hashlib.sha256(
|
||||
json.dumps(unsigned, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
if expected_lock_hash != actual_lock_hash:
|
||||
raise SystemExit("WebUI package lock hash does not match its contents")
|
||||
rows = {}
|
||||
for item in lock["webui"]:
|
||||
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
|
||||
raise SystemExit("WebUI package lock contains a malformed artifact")
|
||||
if item["name"] in rows:
|
||||
raise SystemExit(f"WebUI package lock contains duplicate artifact {item['name']}")
|
||||
rows[item["name"]] = item
|
||||
requested = []
|
||||
for line in output.read_text(encoding="utf-8").splitlines():
|
||||
if not line:
|
||||
continue
|
||||
name, _source_ref = line.split("\t", 1)
|
||||
row = rows.get(name)
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(f"WebUI package lock has no artifact for {name}")
|
||||
filename = row.get("filename")
|
||||
if not isinstance(filename, str) or Path(filename).name != filename:
|
||||
raise SystemExit(f"WebUI package lock has an invalid filename for {name}")
|
||||
artifact = package_dir / filename
|
||||
if artifact.is_symlink() or not artifact.is_file():
|
||||
raise SystemExit(f"WebUI package artifact is missing for {name}")
|
||||
encoded = artifact.read_bytes()
|
||||
if len(encoded) != row.get("size") or hashlib.sha256(encoded).hexdigest() != row.get("sha256"):
|
||||
raise SystemExit(f"WebUI package artifact hash does not match for {name}")
|
||||
requested.append(f"{name}\tfile:{artifact}")
|
||||
output.write_text("\n".join(requested) + "\n", encoding="utf-8")
|
||||
PY
|
||||
fi
|
||||
|
||||
module_paths=()
|
||||
while IFS=$'\t' read -r package_name spec; do
|
||||
[[ -n "${package_name:-}" ]] || continue
|
||||
if [[ "$spec" == file:* ]]; then
|
||||
echo "Installing $package_name from verified package artifact"
|
||||
module_paths+=("$spec")
|
||||
continue
|
||||
fi
|
||||
git_url="${spec%%#*}"
|
||||
git_ref="${spec#*#}"
|
||||
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
|
||||
|
||||
@@ -456,6 +456,13 @@ path.write_text(updated)
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_developer_meta_package() {
|
||||
"$PYTHON" "$META_ROOT/tools/release/generate-developer-meta-package.py" \
|
||||
--workspace "$PARENT" \
|
||||
--requirements "$META_ROOT/requirements-release.txt" \
|
||||
--output "$META_ROOT/packages/govoplan-meta/pyproject.toml"
|
||||
}
|
||||
|
||||
update_version_files() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
@@ -875,8 +882,10 @@ done
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
|
||||
echo "Would synchronize packages/govoplan-meta/pyproject.toml"
|
||||
else
|
||||
update_release_requirements "$TARGET_VERSION"
|
||||
update_developer_meta_package
|
||||
fi
|
||||
|
||||
refresh_development_webui_lock
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download, verify, and lock exact GovOPlaN registry package artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from email.parser import BytesParser
|
||||
from email.policy import compat32
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
import zipfile
|
||||
|
||||
|
||||
NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
WEBUI_NAME = re.compile(r"^@govoplan/[a-z0-9]+(?:-[a-z0-9]+)*-webui$")
|
||||
VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,127}$")
|
||||
COMMIT = re.compile(r"^[0-9a-f]{40}$")
|
||||
MAX_ARTIFACT_BYTES = 512 * 1024 * 1024
|
||||
|
||||
|
||||
class PackageArtifactError(ValueError):
|
||||
"""Registry artifacts do not match the selected package set."""
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--package-set", type=Path, required=True)
|
||||
parser.add_argument("--wheelhouse", type=Path, required=True)
|
||||
parser.add_argument("--webui-packages", type=Path, required=True)
|
||||
parser.add_argument("--lock-output", type=Path, required=True)
|
||||
parser.add_argument("--requirements-output", type=Path)
|
||||
parser.add_argument("--python", default=sys.executable)
|
||||
parser.add_argument("--npm", default="npm")
|
||||
return parser
|
||||
|
||||
|
||||
def resolve(args: argparse.Namespace) -> dict[str, object]:
|
||||
package_set = _load_package_set(args.package_set)
|
||||
wheelhouse = args.wheelhouse.expanduser().resolve()
|
||||
webui_packages = args.webui_packages.expanduser().resolve()
|
||||
_require_empty_destination(wheelhouse)
|
||||
_require_empty_destination(webui_packages)
|
||||
wheelhouse.parent.mkdir(parents=True, exist_ok=True)
|
||||
webui_packages.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-package-resolution-") as value:
|
||||
temporary = Path(value)
|
||||
wheels = temporary / "wheels"
|
||||
webui = temporary / "webui"
|
||||
wheels.mkdir()
|
||||
webui.mkdir()
|
||||
_download_wheels(
|
||||
packages=tuple(package_set["python"]),
|
||||
destination=wheels,
|
||||
python=args.python,
|
||||
index_url=str(package_set["registries"]["python"]),
|
||||
)
|
||||
_download_webui(
|
||||
packages=tuple(package_set["webui"]),
|
||||
destination=webui,
|
||||
npm=args.npm,
|
||||
registry=str(package_set["registries"]["npm"]),
|
||||
)
|
||||
python_rows = _verify_wheels(tuple(package_set["python"]), wheels)
|
||||
webui_rows = _verify_webui(tuple(package_set["webui"]), webui)
|
||||
lock: dict[str, object] = {
|
||||
"schema_version": "1",
|
||||
"release_version": package_set["release_version"],
|
||||
"package_set_sha256": package_set["package_set_sha256"],
|
||||
"registries": package_set["registries"],
|
||||
"python": python_rows,
|
||||
"webui": webui_rows,
|
||||
}
|
||||
lock["lock_sha256"] = _canonical_sha256(lock)
|
||||
shutil.copytree(wheels, wheelhouse, dirs_exist_ok=True)
|
||||
shutil.copytree(webui, webui_packages, dirs_exist_ok=True)
|
||||
args.lock_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.lock_output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
if args.requirements_output is not None:
|
||||
_write_requirements(args.requirements_output, python_rows)
|
||||
return lock
|
||||
|
||||
|
||||
def _load_package_set(path: Path) -> dict[str, object]:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict) or value.get("schema_version") != "1":
|
||||
raise PackageArtifactError("package set has an unsupported shape")
|
||||
expected_hash = value.get("package_set_sha256")
|
||||
unsigned = dict(value)
|
||||
unsigned.pop("package_set_sha256", None)
|
||||
if expected_hash != _canonical_sha256(unsigned):
|
||||
raise PackageArtifactError("package set hash does not match its contents")
|
||||
registries = value.get("registries")
|
||||
if not isinstance(registries, dict) or set(registries) != {"python", "npm"}:
|
||||
raise PackageArtifactError("package set registries are invalid")
|
||||
for registry in registries.values():
|
||||
parsed = urlsplit(str(registry))
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||
raise PackageArtifactError("package registries must use credential-free HTTPS URLs")
|
||||
for group in ("python", "webui"):
|
||||
packages = value.get(group)
|
||||
if not isinstance(packages, list) or not packages:
|
||||
raise PackageArtifactError(f"package set {group} entries are missing")
|
||||
_validate_package_entries(group, packages)
|
||||
return value
|
||||
|
||||
|
||||
def _validate_package_entries(group: str, packages: list[object]) -> None:
|
||||
names: set[str] = set()
|
||||
for raw in packages:
|
||||
if not isinstance(raw, dict):
|
||||
raise PackageArtifactError(f"package set {group} entry is malformed")
|
||||
name = raw.get("name")
|
||||
version = raw.get("version")
|
||||
repository = raw.get("repository")
|
||||
tag = raw.get("tag")
|
||||
commit = raw.get("commit")
|
||||
name_valid = (
|
||||
isinstance(name, str)
|
||||
and (NAME.fullmatch(name) if group == "python" else WEBUI_NAME.fullmatch(name))
|
||||
)
|
||||
if (
|
||||
not name_valid
|
||||
or name in names
|
||||
or not isinstance(version, str)
|
||||
or VERSION.fullmatch(version) is None
|
||||
or not isinstance(repository, str)
|
||||
or NAME.fullmatch(repository) is None
|
||||
or tag != f"v{version}"
|
||||
or not isinstance(commit, str)
|
||||
or COMMIT.fullmatch(commit) is None
|
||||
):
|
||||
raise PackageArtifactError(f"package set {group} entry has an invalid identity")
|
||||
names.add(name)
|
||||
if group == "python":
|
||||
extras = raw.get("extras")
|
||||
if not isinstance(extras, list) or any(
|
||||
not isinstance(item, str)
|
||||
or re.fullmatch(r"[a-z][a-z0-9_-]*", item) is None
|
||||
for item in extras
|
||||
):
|
||||
raise PackageArtifactError("package set Python extras are invalid")
|
||||
|
||||
|
||||
def _download_wheels(
|
||||
*, packages: tuple[dict[str, object], ...], destination: Path, python: str, index_url: str
|
||||
) -> None:
|
||||
requirements = [_python_requirement(item) for item in packages]
|
||||
environment = dict(os.environ)
|
||||
environment["PIP_INDEX_URL"] = _authenticated_url(index_url)
|
||||
environment["PIP_EXTRA_INDEX_URL"] = ""
|
||||
environment["PIP_CONFIG_FILE"] = os.devnull
|
||||
environment["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
||||
subprocess.run(
|
||||
[python, "-m", "pip", "download", "--no-deps", "--only-binary=:all:", "--dest", str(destination), *requirements],
|
||||
check=True,
|
||||
env=environment,
|
||||
)
|
||||
|
||||
|
||||
def _download_webui(
|
||||
*, packages: tuple[dict[str, object], ...], destination: Path, npm: str, registry: str
|
||||
) -> None:
|
||||
environment = dict(os.environ)
|
||||
npmrc: tempfile.NamedTemporaryFile[bytes] | None = None
|
||||
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
|
||||
if token:
|
||||
parsed = urlsplit(registry)
|
||||
auth_path = f"//{parsed.netloc}{parsed.path}:_authToken={token}\n"
|
||||
npmrc = tempfile.NamedTemporaryFile(prefix="govoplan-npmrc-", delete=False)
|
||||
npmrc.write(f"@govoplan:registry={registry}\n{auth_path}".encode("utf-8"))
|
||||
npmrc.close()
|
||||
os.chmod(npmrc.name, 0o600)
|
||||
environment["NPM_CONFIG_USERCONFIG"] = npmrc.name
|
||||
try:
|
||||
for item in packages:
|
||||
subprocess.run(
|
||||
[npm, "pack", f"{item['name']}@{item['version']}", "--ignore-scripts", "--pack-destination", str(destination), "--registry", registry],
|
||||
check=True,
|
||||
env=environment,
|
||||
)
|
||||
finally:
|
||||
if npmrc is not None:
|
||||
Path(npmrc.name).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _verify_wheels(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||
expected = {_normalize(str(item["name"])): item for item in packages}
|
||||
rows: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
for path in sorted(root.glob("*.whl")):
|
||||
identity = _wheel_identity(path)
|
||||
name = str(identity["name"])
|
||||
package = expected.get(name)
|
||||
if package is None or identity["version"] != package["version"] or name in seen:
|
||||
raise PackageArtifactError(f"unexpected wheel artifact: {path.name}")
|
||||
seen.add(name)
|
||||
rows.append(_artifact_row(path, package))
|
||||
if seen != set(expected):
|
||||
raise PackageArtifactError("registry did not return every selected Python wheel")
|
||||
return sorted(rows, key=lambda item: str(item["name"]))
|
||||
|
||||
|
||||
def _verify_webui(packages: tuple[dict[str, object], ...], root: Path) -> list[dict[str, object]]:
|
||||
expected = {str(item["name"]): item for item in packages}
|
||||
rows: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
for path in sorted(root.glob("*.tgz")):
|
||||
identity = _npm_identity(path)
|
||||
name = str(identity["name"])
|
||||
package = expected.get(name)
|
||||
if package is None or identity["version"] != package["version"] or name in seen:
|
||||
raise PackageArtifactError(f"unexpected WebUI artifact: {path.name}")
|
||||
seen.add(name)
|
||||
row = _artifact_row(path, package)
|
||||
row["integrity"] = "sha512-" + base64.b64encode(hashlib.sha512(path.read_bytes()).digest()).decode("ascii")
|
||||
rows.append(row)
|
||||
if seen != set(expected):
|
||||
raise PackageArtifactError("registry did not return every selected WebUI package")
|
||||
return sorted(rows, key=lambda item: str(item["name"]))
|
||||
|
||||
|
||||
def _wheel_identity(path: Path) -> dict[str, str]:
|
||||
_bounded(path)
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
metadata = [
|
||||
item for item in archive.infolist()
|
||||
if PurePosixPath(item.filename).name == "METADATA"
|
||||
and PurePosixPath(item.filename).parent.name.endswith(".dist-info")
|
||||
]
|
||||
if len(metadata) != 1 or metadata[0].file_size > 1024 * 1024:
|
||||
raise PackageArtifactError(f"wheel metadata is invalid: {path.name}")
|
||||
parsed = BytesParser(policy=compat32).parsebytes(archive.read(metadata[0]))
|
||||
name = _normalize(str(parsed.get("Name") or ""))
|
||||
version = str(parsed.get("Version") or "")
|
||||
if NAME.fullmatch(name) is None or VERSION.fullmatch(version) is None:
|
||||
raise PackageArtifactError(f"wheel identity is invalid: {path.name}")
|
||||
return {"name": name, "version": version}
|
||||
|
||||
|
||||
def _npm_identity(path: Path) -> dict[str, str]:
|
||||
_bounded(path)
|
||||
with tarfile.open(path, mode="r:gz") as archive:
|
||||
try:
|
||||
member = archive.getmember("package/package.json")
|
||||
except KeyError as exc:
|
||||
raise PackageArtifactError(f"npm package metadata is missing: {path.name}") from exc
|
||||
if not member.isfile() or member.size > 1024 * 1024:
|
||||
raise PackageArtifactError(f"npm package metadata is invalid: {path.name}")
|
||||
extracted = archive.extractfile(member)
|
||||
if extracted is None:
|
||||
raise PackageArtifactError(f"npm package metadata cannot be read: {path.name}")
|
||||
value = json.load(extracted)
|
||||
name = value.get("name")
|
||||
version = value.get("version")
|
||||
if not isinstance(name, str) or not name.startswith("@govoplan/") or not isinstance(version, str) or VERSION.fullmatch(version) is None:
|
||||
raise PackageArtifactError(f"npm package identity is invalid: {path.name}")
|
||||
return {"name": name, "version": version}
|
||||
|
||||
|
||||
def _artifact_row(path: Path, package: dict[str, object]) -> dict[str, object]:
|
||||
row = {
|
||||
"name": package["name"],
|
||||
"version": package["version"],
|
||||
"repository": package["repository"],
|
||||
"tag": package["tag"],
|
||||
"commit": package["commit"],
|
||||
"filename": path.name,
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"size": path.stat().st_size,
|
||||
}
|
||||
if "extras" in package:
|
||||
row["extras"] = package["extras"]
|
||||
return row
|
||||
|
||||
|
||||
def _write_requirements(path: Path, rows: list[dict[str, object]]) -> None:
|
||||
lines = ["--no-index", "--find-links ./local-wheels", "--require-hashes"]
|
||||
for row in rows:
|
||||
selected_extras = row.get("extras") or []
|
||||
extras = (
|
||||
f"[{','.join(str(value) for value in selected_extras)}]"
|
||||
if selected_extras
|
||||
else ""
|
||||
)
|
||||
lines.append(
|
||||
f"{row['name']}{extras}=={row['version']} "
|
||||
f"--hash=sha256:{row['sha256']}"
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _python_requirement(item: dict[str, object]) -> str:
|
||||
extras = item.get("extras") or []
|
||||
suffix = f"[{','.join(str(value) for value in extras)}]" if extras else ""
|
||||
return f"{item['name']}{suffix}=={item['version']}"
|
||||
|
||||
|
||||
def _authenticated_url(url: str) -> str:
|
||||
token = os.environ.get("GOVOPLAN_PACKAGE_TOKEN", "")
|
||||
username = os.environ.get("GOVOPLAN_PACKAGE_USERNAME", "")
|
||||
if not token:
|
||||
return url
|
||||
if not username:
|
||||
raise PackageArtifactError("GOVOPLAN_PACKAGE_USERNAME is required with a package token")
|
||||
parsed = urlsplit(url)
|
||||
return urlunsplit(
|
||||
(
|
||||
parsed.scheme,
|
||||
f"{quote(username, safe='')}:{quote(token, safe='')}@{parsed.netloc}",
|
||||
parsed.path,
|
||||
parsed.query,
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _require_empty_destination(path: Path) -> None:
|
||||
if path.exists() and (not path.is_dir() or any(path.iterdir())):
|
||||
raise PackageArtifactError(f"output directory must be absent or empty: {path}")
|
||||
if path.is_symlink():
|
||||
raise PackageArtifactError(f"output directory must not be a symlink: {path}")
|
||||
|
||||
|
||||
def _bounded(path: Path) -> None:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_ARTIFACT_BYTES:
|
||||
raise PackageArtifactError(f"package artifact is invalid or too large: {path.name}")
|
||||
|
||||
|
||||
def _normalize(value: str) -> str:
|
||||
return re.sub(r"[-_.]+", "-", value.strip().lower())
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
try:
|
||||
lock = resolve(args)
|
||||
except (PackageArtifactError, OSError, ValueError, subprocess.CalledProcessError, zipfile.BadZipFile, tarfile.TarError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"Resolved {len(lock['python'])} Python and {len(lock['webui'])} WebUI packages.")
|
||||
print(f"Package artifact lock written to {args.lock_output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user