Harden catalog publication transaction

This commit is contained in:
2026-07-22 21:20:08 +02:00
parent fdee766993
commit 349a099e5a
4 changed files with 1807 additions and 178 deletions

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import subprocess
import sys
@@ -15,8 +16,15 @@ if str(RELEASE_TOOLS_ROOT) not in sys.path:
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
from govoplan_release.publisher import ( # noqa: E402
FrozenPublicationRemote,
bind_publication_remote,
commit_publication_tree,
publication_mutation_trust_issues,
publish_catalog_candidate,
remote_publication_identity,
verify_committed_publication,
verify_remote_branch_head,
verify_remote_publication,
)
@@ -26,13 +34,17 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
web_root = Path(tmp) / "website"
module_root = web_root / "public" / "catalogs" / "v1" / "modules"
module_root.mkdir(parents=True)
catalog = web_root / "public" / "catalogs" / "v1" / "channels" / "stable.json"
catalog = (
web_root / "public" / "catalogs" / "v1" / "channels" / "stable.json"
)
keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
catalog.parent.mkdir(parents=True)
expected = {
catalog.relative_to(web_root).as_posix(): b'{"catalog":true}\n',
keyring.relative_to(web_root).as_posix(): b'{"keys":[]}\n',
(module_root / "index.json").relative_to(web_root).as_posix(): b'{"modules":[]}\n',
(module_root / "index.json")
.relative_to(web_root)
.as_posix(): b'{"modules":[]}\n',
}
for relative, encoded in expected.items():
target = web_root / relative
@@ -72,7 +84,339 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
module_root=module_root,
)
def test_apply_writes_validated_objects_even_if_candidate_path_changes(self) -> None:
def test_private_index_commit_has_one_parent_and_only_computed_delta(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
web_root = root / "website"
module_root = web_root / "public" / "catalogs" / "v1" / "modules"
channel = (
web_root / "public" / "catalogs" / "v1" / "channels" / "stable.json"
)
keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
old_module = module_root / "obsolete.json"
unrelated = web_root / "unrelated.txt"
for path, encoded in (
(channel, b'{"old":true}\n'),
(keyring, b'{"keys":[]}\n'),
(old_module, b'{"obsolete":true}\n'),
(unrelated, b"keep\n"),
):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(encoded)
self._git(web_root, "init", "--quiet")
self._git(web_root, "config", "user.email", "test@example.test")
self._git(web_root, "config", "user.name", "Test")
self._git(web_root, "add", ".")
self._git(web_root, "commit", "--quiet", "-m", "base")
frozen_head = self._git(web_root, "rev-parse", "HEAD").strip()
branch = self._git(web_root, "branch", "--show-current").strip()
malicious = web_root / "not-in-publication.txt"
malicious.write_text("staged but excluded\n", encoding="utf-8")
self._git(web_root, "add", malicious.name)
marker = root / "reference-hook-ran"
hook = web_root / ".git" / "hooks" / "reference-transaction"
hook.write_text(f"#!/bin/sh\ntouch {marker}\n", encoding="utf-8")
hook.chmod(0o755)
expected = {
channel.relative_to(web_root).as_posix(): b'{"new":true}\n',
keyring.relative_to(web_root).as_posix(): b'{"keys":["release"]}\n',
(module_root / "index.json")
.relative_to(web_root)
.as_posix(): b'{"modules":[]}\n',
}
redirected = root / "attacker-index"
with patch.dict(
os.environ,
{
"GIT_DIR": str(root / "attacker-git-dir"),
"GIT_INDEX_FILE": str(redirected),
"GIT_OBJECT_DIRECTORY": str(root / "attacker-objects"),
"GIT_CONFIG_GLOBAL": str(root / "attacker-config"),
},
):
commit_sha = commit_publication_tree(
web_root=web_root,
frozen_head=frozen_head,
branch=branch,
expected_blobs=expected,
module_root=module_root,
message="Exact publication",
)
parents = self._git(
web_root, "rev-list", "--parents", "-n", "1", commit_sha
).split()
changed = set(
filter(
None,
self._git(
web_root,
"diff-tree",
"--no-commit-id",
"--name-only",
"-r",
frozen_head,
commit_sha,
).splitlines(),
)
)
hook_ran = marker.exists()
inherited_index_used = redirected.exists()
commit_paths = self._git(
web_root, "ls-tree", "-r", "--name-only", commit_sha
)
self.assertEqual([commit_sha, frozen_head], parents)
self.assertEqual(
{
"public/catalogs/v1/channels/stable.json",
"public/catalogs/v1/keyring.json",
"public/catalogs/v1/modules/index.json",
"public/catalogs/v1/modules/obsolete.json",
},
changed,
)
self.assertFalse(hook_ran)
self.assertFalse(inherited_index_used)
self.assertNotIn("not-in-publication.txt", commit_paths)
def test_remote_binding_and_annotated_publication_identity_are_exact(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
remote_root = root / "website.git"
remote_root.mkdir()
self._git(remote_root, "init", "--bare", "--quiet")
web_root = root / "website"
web_root.mkdir()
self._git(web_root, "init", "--quiet")
self._git(web_root, "config", "user.email", "test@example.test")
self._git(web_root, "config", "user.name", "Test")
self._git(web_root, "branch", "-M", "main")
web_root.joinpath("base.txt").write_text("base\n", encoding="utf-8")
self._git(web_root, "add", ".")
self._git(web_root, "commit", "--quiet", "-m", "base")
self._git(web_root, "remote", "add", "origin", str(remote_root))
self._git(web_root, "push", "--quiet", "-u", "origin", "main")
base_commit = self._git(web_root, "rev-parse", "HEAD").strip()
frozen = bind_publication_remote(
web_root=web_root,
remote="origin",
registered_remote=str(remote_root),
)
self.assertIsInstance(frozen, FrozenPublicationRemote)
verify_remote_branch_head(
web_root=web_root,
remote_url=frozen.url,
branch="main",
expected_commit=base_commit,
)
web_root.joinpath("catalog.json").write_text("{}\n", encoding="utf-8")
self._git(web_root, "add", "catalog.json")
self._git(web_root, "commit", "--quiet", "-m", "publication")
commit_sha = self._git(web_root, "rev-parse", "HEAD").strip()
tag_name = "catalog-test"
self._git(web_root, "tag", "-a", tag_name, "-m", "publication")
tag_object = self._git(
web_root, "rev-parse", f"refs/tags/{tag_name}"
).strip()
self._git(
web_root,
"push",
"--quiet",
"--atomic",
"origin",
f"{commit_sha}:refs/heads/main",
f"{tag_object}:refs/tags/{tag_name}",
)
identity = remote_publication_identity(
web_root=web_root,
remote_url=frozen.url,
branch="main",
tag_name=tag_name,
)
verified = verify_remote_publication(
web_root=web_root,
remote_url=frozen.url,
branch="main",
tag_name=tag_name,
expected_commit=commit_sha,
expected_tag_object=tag_object,
)
self._git(
web_root,
"remote",
"set-url",
"--add",
"--push",
"origin",
str(root / "other.git"),
)
with self.assertRaisesRegex(RuntimeError, "do not match"):
bind_publication_remote(
web_root=web_root,
remote="origin",
registered_remote=str(remote_root),
)
self.assertEqual(identity, verified)
self.assertEqual(commit_sha, identity["publication_commit_sha"])
self.assertEqual(tag_object, identity["publication_tag_object_sha"])
self.assertEqual(commit_sha, identity["publication_tag_commit_sha"])
def test_mutation_rejects_writable_website_and_git_configuration(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
web_root = root / "website"
web_root.mkdir()
self._git(web_root, "init", "--quiet")
candidate = self._candidate(root, key="trusted-key")
target_root = web_root / "public" / "catalogs" / "v1"
target_catalog = target_root / "channels" / "stable.json"
target_keyring = target_root / "keyring.json"
target_keyring.parent.mkdir(parents=True)
target_keyring.write_text("{}\n", encoding="utf-8")
web_root.chmod(0o777)
root_issues = publication_mutation_trust_issues(
web_root=web_root,
candidate_catalog=candidate / "channels" / "stable.json",
candidate_keyring=candidate / "keyring.json",
target_catalog=target_catalog,
target_keyring=target_keyring,
target_modules=target_root / "modules",
)
web_root.chmod(0o755)
config = web_root / ".git" / "config"
config.chmod(0o666)
config_issues = publication_mutation_trust_issues(
web_root=web_root,
candidate_catalog=candidate / "channels" / "stable.json",
candidate_keyring=candidate / "keyring.json",
target_catalog=target_catalog,
target_keyring=target_keyring,
target_modules=target_root / "modules",
)
self.assertIn("website root is writable by another user", root_issues)
self.assertIn("website Git config is writable by another user", config_issues)
def test_push_returns_only_independently_verified_publication_receipt(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
candidate = self._candidate(root, key="trusted-key")
catalog_path = candidate / "channels" / "stable.json"
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
catalog["sequence"] = 7
catalog["core_release"]["python_package"] = "govoplan-core"
catalog["release"] = {
"selected_units": [
{
"repo": "govoplan-core",
"version": "1.2.3",
"tag": "v1.2.3",
"commit_sha": "a" * 40,
"tag_object_sha": "b" * 40,
}
],
"artifacts": [
{
"artifact_kind": "python-wheel",
"package_name": "govoplan-core",
"package_version": "1.2.3",
"archive_sha256": "c" * 64,
"archive_size": 100,
"installed_payload": {
"algorithm": "govoplan-wheel-declared-payload-v1",
"sha256": "d" * 64,
"file_count": 1,
},
"requires_installer_receipt": True,
}
],
}
catalog["signatures"] = [{}]
catalog_path.write_text(json.dumps(catalog), encoding="utf-8")
remote_root = root / "website.git"
remote_root.mkdir()
self._git(remote_root, "init", "--bare", "--quiet")
web_root = root / "website"
target_keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
target_keyring.parent.mkdir(parents=True)
target_keyring.write_text(json.dumps(self._keyring("trusted-key")))
self._git(web_root, "init", "--quiet")
self._git(web_root, "config", "user.email", "test@example.test")
self._git(web_root, "config", "user.name", "Test")
self._git(web_root, "branch", "-M", "main")
self._git(web_root, "remote", "add", "origin", str(remote_root))
self._git(web_root, "add", ".")
self._git(web_root, "commit", "--quiet", "-m", "base")
self._git(web_root, "push", "--quiet", "-u", "origin", "main")
frozen_head = self._git(web_root, "rev-parse", "HEAD").strip()
frozen_remote = bind_publication_remote(
web_root=web_root,
remote="origin",
registered_remote=str(remote_root),
)
with (
patch(
"govoplan_release.publisher.validate_module_package_catalog",
return_value={"valid": True, "warnings": [], "error": None},
),
patch(
"govoplan_release.publisher.source_tag_provenance_issues",
return_value=(),
),
patch(
"govoplan_release.publisher.publication_runtime_trust_issues",
return_value=(),
),
patch(
"govoplan_release.publisher.registered_website_remote",
return_value=str(remote_root),
),
):
result = publish_catalog_candidate(
candidate_dir=candidate,
web_root=web_root,
workspace_root=root,
apply=True,
push=True,
branch="main",
tag_name="catalog-test",
expected_website_head=frozen_head,
expected_website_branch="main",
expected_remote_sha256=frozen_remote.sha256,
)
remote_identity = remote_publication_identity(
web_root=web_root,
remote_url=str(remote_root),
branch="main",
tag_name="catalog-test",
)
self.assertEqual("published", result.status)
self.assertEqual("origin", result.remote)
self.assertEqual(
remote_identity["publication_commit_sha"], result.publication_commit_sha
)
self.assertEqual(
remote_identity["publication_tag_object_sha"],
result.publication_tag_object_sha,
)
self.assertEqual(
remote_identity["publication_tag_commit_sha"],
result.publication_tag_commit_sha,
)
def test_apply_writes_validated_objects_even_if_candidate_path_changes(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
candidate = self._candidate(root, key="trusted-key")
@@ -112,7 +456,13 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
target_keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
target_keyring.parent.mkdir(parents=True)
target_keyring.write_text(json.dumps(self._keyring("trusted-key")))
(web_root / ".git").mkdir()
registered_remote = "ssh://example.test/release/website.git"
self._git(web_root, "init", "--quiet")
self._git(web_root, "config", "user.email", "test@example.test")
self._git(web_root, "config", "user.name", "Test")
self._git(web_root, "remote", "add", "origin", registered_remote)
self._git(web_root, "add", ".")
self._git(web_root, "commit", "--quiet", "-m", "base")
def validate_and_swap(*args, **kwargs):
del args, kwargs
@@ -139,6 +489,14 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
return_value=(),
),
patch("govoplan_release.publisher.website_dirty", return_value=False),
patch(
"govoplan_release.publisher.publication_runtime_trust_issues",
return_value=(),
),
patch(
"govoplan_release.publisher.registered_website_remote",
return_value=registered_remote,
),
):
result = publish_catalog_candidate(
candidate_dir=candidate,
@@ -150,12 +508,7 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
published = json.loads(
(
web_root
/ "public"
/ "catalogs"
/ "v1"
/ "channels"
/ "stable.json"
web_root / "public" / "catalogs" / "v1" / "channels" / "stable.json"
).read_text(encoding="utf-8")
)
@@ -274,7 +627,9 @@ class ReleaseCatalogPublicationTests(unittest.TestCase):
}
)
)
candidate.joinpath("keyring.json").write_text(json.dumps(ReleaseCatalogPublicationTests._keyring(key)))
candidate.joinpath("keyring.json").write_text(
json.dumps(ReleaseCatalogPublicationTests._keyring(key))
)
return candidate
@staticmethod

View File

@@ -334,6 +334,7 @@ class CatalogPublishResult:
target_keyring_path: str
branch: str | None
tag_name: str | None
remote: str
validation_valid: bool
validation_error: str | None = None
validation_warnings: tuple[str, ...] = ()
@@ -343,6 +344,9 @@ class CatalogPublishResult:
target_keyring_hash_before: str | None = None
catalog_changed: bool = False
keyring_changed: bool = False
publication_commit_sha: str | None = None
publication_tag_object_sha: str | None = None
publication_tag_commit_sha: str | None = None
steps: tuple[CatalogPublishStep, ...] = ()
notes: tuple[str, ...] = ()

File diff suppressed because it is too large Load Diff

View File

@@ -9,7 +9,10 @@ from pathlib import Path
from govoplan_release.module_directory import write_module_directory
from govoplan_release.model import to_jsonable
from govoplan_release.publisher import publish_catalog_candidate
from govoplan_release.publisher import (
publication_runtime_trust_issues,
publish_catalog_candidate,
)
from govoplan_release.selective_catalog import build_selective_catalog_candidate
from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT
@@ -18,11 +21,27 @@ def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
selective = subparsers.add_parser("selective", help="Build a signed selective channel catalog candidate.")
selective.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT)
selective = subparsers.add_parser(
"selective", help="Build a signed selective channel catalog candidate."
)
selective.add_argument(
"--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT
)
selective.add_argument("--channel", default="stable")
selective.add_argument("--repo-version", action="append", default=[], metavar="REPO=VERSION", required=True)
selective.add_argument("--catalog-signing-key", action="append", default=[], metavar="KEY_ID=PRIVATE_KEY", required=True)
selective.add_argument(
"--repo-version",
action="append",
default=[],
metavar="REPO=VERSION",
required=True,
)
selective.add_argument(
"--catalog-signing-key",
action="append",
default=[],
metavar="KEY_ID=PRIVATE_KEY",
required=True,
)
selective.add_argument(
"--python-artifact",
action="append",
@@ -56,41 +75,76 @@ def main() -> int:
),
)
selective.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
selective.add_argument("--repository-base", default="git+ssh://git@git.add-ideas.de/add-ideas")
selective.add_argument("--source-remote", default="origin", help="Configured Git remote containing immutable source tags.")
selective.add_argument(
"--repository-base", default="git+ssh://git@git.add-ideas.de/add-ideas"
)
selective.add_argument(
"--source-remote",
default="origin",
help="Configured Git remote containing immutable source tags.",
)
selective.add_argument("--expires-days", type=int, default=90)
selective.add_argument("--sequence", type=int)
selective.add_argument("--skip-public-check", action="store_true")
selective.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.")
selective.add_argument(
"--json", action="store_true", help="Print machine-readable summary JSON."
)
publish = subparsers.add_parser("publish-candidate", help="Publish a reviewed catalog candidate into the website repo.")
publish = subparsers.add_parser(
"publish-candidate",
help="Publish a reviewed catalog candidate into the website repo.",
)
publish.add_argument("--candidate-dir", type=Path, required=True)
publish.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT)
publish.add_argument("--web-root", type=Path)
publish.add_argument("--channel", default="stable")
publish.add_argument("--apply", action="store_true", help="Copy candidate files into the website repo. Without this, only preview.")
publish.add_argument(
"--apply",
action="store_true",
help="Copy candidate files into the website repo. Without this, only preview.",
)
publish.add_argument("--build-web", action="store_true")
publish.add_argument("--commit", action="store_true")
publish.add_argument("--tag", action="store_true", help="Create a website catalog publication tag. Implies --commit.")
publish.add_argument("--push", action="store_true", help="Push the website branch and tag. Implies --commit.")
publish.add_argument(
"--tag",
action="store_true",
help="Create a website catalog publication tag. Implies --commit.",
)
publish.add_argument(
"--push",
action="store_true",
help="Push the website branch and tag. Implies --commit.",
)
publish.add_argument("--remote", default="origin")
publish.add_argument("--source-remote", default="origin", help="Configured Git remote containing catalog source tags.")
publish.add_argument(
"--source-remote",
default="origin",
help="Configured Git remote containing catalog source tags.",
)
publish.add_argument("--branch")
publish.add_argument("--tag-name")
publish.add_argument("--npm", default="npm")
publish.add_argument("--allow-dirty-website", action="store_true")
publish.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.")
publish.add_argument(
"--json", action="store_true", help="Print machine-readable summary JSON."
)
directory = subparsers.add_parser("module-directory", help="Generate browsable module-directory files from a catalog and keyring.")
directory = subparsers.add_parser(
"module-directory",
help="Generate browsable module-directory files from a catalog and keyring.",
)
directory.add_argument("--catalog", type=Path, required=True)
directory.add_argument("--keyring", type=Path, required=True)
directory.add_argument("--output-dir", type=Path, required=True)
directory.add_argument("--channel", default="stable")
directory.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
directory.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.")
directory.add_argument(
"--json", action="store_true", help="Print machine-readable summary JSON."
)
args = parser.parse_args()
if args.command == "selective":
require_release_runtime_trust()
result = build_selective_catalog_candidate(
repo_versions=parse_repo_versions(tuple(args.repo_version)),
channel=args.channel,
@@ -114,6 +168,8 @@ def main() -> int:
print_text_summary(payload)
return 0 if result.status == "ready" else 1
if args.command == "publish-candidate":
if args.apply:
require_release_runtime_trust()
result = publish_catalog_candidate(
candidate_dir=args.candidate_dir,
workspace_root=args.workspace_root,
@@ -138,6 +194,7 @@ def main() -> int:
print_publish_summary(payload)
return 0 if result.status in {"ready", "applied", "published"} else 1
if args.command == "module-directory":
require_release_runtime_trust()
catalog_payload = json.loads(args.catalog.read_text(encoding="utf-8"))
keyring_payload = json.loads(args.keyring.read_text(encoding="utf-8"))
files = write_module_directory(
@@ -147,7 +204,11 @@ def main() -> int:
channel=args.channel,
public_base_url=args.public_base_url,
)
payload = {"status": "generated", "output_dir": str(args.output_dir), "files": [str(path) for path in files]}
payload = {
"status": "generated",
"output_dir": str(args.output_dir),
"files": [str(path) for path in files],
}
if args.json:
print(json.dumps(payload, indent=2, sort_keys=True))
else:
@@ -162,6 +223,12 @@ def main() -> int:
return 2
def require_release_runtime_trust() -> None:
issues = publication_runtime_trust_issues()
if issues:
raise SystemExit("Release tooling trust gate failed: " + "; ".join(issues))
def parse_repo_versions(values: tuple[str, ...]) -> dict[str, str]:
result: dict[str, str] = {}
for value in values:
@@ -185,7 +252,9 @@ def parse_repo_paths(values: tuple[str, ...]) -> dict[str, Path]:
repo = repo.strip()
path_text = path_text.strip()
if not repo or not path_text or repo in result:
raise SystemExit(f"--python-artifact must uniquely use REPO=/path/to/wheel: {value}")
raise SystemExit(
f"--python-artifact must uniquely use REPO=/path/to/wheel: {value}"
)
result[repo] = Path(path_text).expanduser()
return result