From 9788bdde0c325e93599e4412ee05bbc7b979c2e4 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 23 Jul 2026 00:09:43 +0200 Subject: [PATCH] ci: harden repository bootstrap transport --- repositories.json | 2 +- tests/test_repository_bootstrap.py | 364 +++++++++++++++++++++++++++ tools/repo/bootstrap-repositories.py | 119 ++++++++- 3 files changed, 475 insertions(+), 10 deletions(-) create mode 100644 tests/test_repository_bootstrap.py diff --git a/repositories.json b/repositories.json index ac8b8f3..66ac259 100644 --- a/repositories.json +++ b/repositories.json @@ -64,7 +64,7 @@ {"name": "govoplan-templates", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-templates.git", "path": "govoplan-templates"}, {"name": "govoplan-tenancy", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-tenancy.git", "path": "govoplan-tenancy"}, {"name": "govoplan-transparency", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-transparency.git", "path": "govoplan-transparency"}, - {"name": "addideas-govoplan-website", "category": "website", "subtype": "public-site", "remote": "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", "path": "addideas-govoplan-website"}, + {"name": "addideas-govoplan-website", "category": "website", "subtype": "public-site", "remote": "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", "path": "addideas-govoplan-website", "bootstrap_transport": "registered"}, {"name": "govoplan-workflow", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-workflow.git", "path": "govoplan-workflow"}, {"name": "govoplan-xoev", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xoev.git", "path": "govoplan-xoev"}, {"name": "govoplan-xrechnung", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xrechnung.git", "path": "govoplan-xrechnung"}, diff --git a/tests/test_repository_bootstrap.py b/tests/test_repository_bootstrap.py new file mode 100644 index 0000000..7aaf5cc --- /dev/null +++ b/tests/test_repository_bootstrap.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + + +META_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = META_ROOT / "tools" / "repo" / "bootstrap-repositories.py" + + +def load_bootstrap_module(): + spec = importlib.util.spec_from_file_location("bootstrap_repositories", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class RepositoryBootstrapTests(unittest.TestCase): + def test_public_https_transport_rewrites_registered_gitea_remotes(self) -> None: + bootstrap = load_bootstrap_module() + + self.assertEqual( + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + bootstrap.clone_remote( + "git@git.add-ideas.de:add-ideas/govoplan-core.git", + transport=bootstrap.PUBLIC_HTTPS_TRANSPORT, + ), + ) + self.assertEqual( + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + bootstrap.clone_remote( + "ssh://git@git.add-ideas.de/add-ideas/govoplan-core.git", + transport=bootstrap.PUBLIC_HTTPS_TRANSPORT, + ), + ) + self.assertEqual( + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + bootstrap.clone_remote( + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + transport=bootstrap.PUBLIC_HTTPS_TRANSPORT, + ), + ) + + def test_registered_transport_preserves_the_manifest_remote(self) -> None: + bootstrap = load_bootstrap_module() + remote = "git@example.test:private/repository.git" + + self.assertEqual( + remote, + bootstrap.clone_remote( + remote, + transport=bootstrap.REGISTERED_TRANSPORT, + ), + ) + + def test_public_https_transport_fails_closed_for_other_hosts(self) -> None: + bootstrap = load_bootstrap_module() + + unsafe_remotes = ( + "git@example.test:add-ideas/govoplan-core.git", + "https://token@git.add-ideas.de/add-ideas/govoplan-core.git", + "https://git.add-ideas.de:443/add-ideas/govoplan-core.git", + "https://git.add-ideas.de/add-ideas/../govoplan-core.git", + "https://git.add-ideas.de/add-ideas/govoplan-core.git?ref=main", + "ssh://root@git.add-ideas.de/add-ideas/govoplan-core.git", + ) + for remote in unsafe_remotes: + with self.subTest(remote=remote), self.assertRaises(ValueError): + bootstrap.clone_remote( + remote, + transport=bootstrap.PUBLIC_HTTPS_TRANSPORT, + ) + + def test_main_clones_missing_repositories_over_public_https(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": "govoplan-core", + "path": "govoplan-core", + "remote": ( + "git@git.add-ideas.de:" + "add-ideas/govoplan-core.git" + ), + } + ], + } + ), + encoding="utf-8", + ) + with ( + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + ): + status = bootstrap.main( + [ + "--parent", + str(parent), + "--transport", + bootstrap.PUBLIC_HTTPS_TRANSPORT, + ] + ) + + self.assertEqual(0, status) + runner.assert_called_once_with( + [ + "git", + "-c", + "credential.helper=", + "clone", + "--", + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + str(parent / "govoplan-core"), + ], + check=True, + env=runner.call_args.kwargs["env"], + ) + environment = runner.call_args.kwargs["env"] + self.assertEqual("/bin/false", environment["GIT_ASKPASS"]) + self.assertEqual("0", environment["GIT_TERMINAL_PROMPT"]) + + def test_main_validates_every_remote_before_cloning(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": "govoplan-core", + "path": "govoplan-core", + "remote": ( + "git@git.add-ideas.de:" + "add-ideas/govoplan-core.git" + ), + }, + { + "name": "unsafe", + "path": "unsafe", + "remote": "git@example.test:private/unsafe.git", + }, + ], + } + ), + encoding="utf-8", + ) + with ( + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + self.assertRaises(ValueError), + ): + bootstrap.main( + [ + "--parent", + str(parent), + "--transport", + bootstrap.PUBLIC_HTTPS_TRANSPORT, + ] + ) + + runner.assert_not_called() + + def test_main_preserves_an_explicit_private_repository_transport(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": "website", + "path": "website", + "remote": ( + "git@git.add-ideas.de:" + "add-ideas/addideas-govoplan-website.git" + ), + "bootstrap_transport": ( + bootstrap.REGISTERED_TRANSPORT + ), + } + ], + } + ), + encoding="utf-8", + ) + with ( + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + ): + status = bootstrap.main( + [ + "--parent", + str(parent), + "--transport", + bootstrap.PUBLIC_HTTPS_TRANSPORT, + ] + ) + + self.assertEqual(0, status) + command = runner.call_args.args[0] + self.assertEqual( + "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", + command[-2], + ) + + def test_main_limits_bootstrap_to_selected_repositories(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": name, + "path": name, + "remote": ( + "git@git.add-ideas.de:add-ideas/" + f"{name}.git" + ), + } + for name in ("govoplan-core", "govoplan-poll") + ], + } + ), + encoding="utf-8", + ) + for repository_filter in ( + ["--repo", "govoplan-core"], + ["--exclude-repo", "govoplan-poll"], + ): + with ( + self.subTest(repository_filter=repository_filter), + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + ): + status = bootstrap.main( + [ + "--parent", + str(parent), + "--transport", + bootstrap.PUBLIC_HTTPS_TRANSPORT, + *repository_filter, + ] + ) + self.assertEqual(0, status) + runner.assert_called_once() + self.assertEqual( + "https://git.add-ideas.de/add-ideas/govoplan-core.git", + runner.call_args.args[0][-2], + ) + + def test_main_rejects_unknown_or_conflicting_repository_filters(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": "govoplan-core", + "path": "govoplan-core", + "remote": ( + "git@git.add-ideas.de:" + "add-ideas/govoplan-core.git" + ), + } + ], + } + ), + encoding="utf-8", + ) + with ( + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + ): + with self.assertRaisesRegex(ValueError, "unknown registered"): + bootstrap.main(["--repo", "govoplan-missing"]) + with self.assertRaisesRegex(ValueError, "selected and excluded"): + bootstrap.main( + [ + "--repo", + "govoplan-core", + "--exclude-repo", + "govoplan-core", + ] + ) + + runner.assert_not_called() + + def test_check_reports_missing_without_cloning(self) -> None: + bootstrap = load_bootstrap_module() + with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory: + root = Path(directory) + parent = root / "checkouts" + parent.mkdir() + root.joinpath("repositories.json").write_text( + json.dumps( + { + "default_parent": str(parent), + "repositories": [ + { + "name": "govoplan-core", + "path": "govoplan-core", + "remote": ( + "git@git.add-ideas.de:" + "add-ideas/govoplan-core.git" + ), + } + ], + } + ), + encoding="utf-8", + ) + with ( + patch.object(bootstrap, "ROOT", root), + patch.object(bootstrap.subprocess, "run") as runner, + ): + status = bootstrap.main( + [ + "--check", + "--parent", + str(parent), + "--transport", + bootstrap.PUBLIC_HTTPS_TRANSPORT, + ] + ) + + self.assertEqual(1, status) + runner.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/repo/bootstrap-repositories.py b/tools/repo/bootstrap-repositories.py index 00fe47a..1eba3a2 100644 --- a/tools/repo/bootstrap-repositories.py +++ b/tools/repo/bootstrap-repositories.py @@ -3,35 +3,136 @@ from __future__ import annotations import argparse import json +import os +import re import subprocess import sys from pathlib import Path +from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[2] +REGISTERED_TRANSPORT = "registered" +PUBLIC_HTTPS_TRANSPORT = "public-https" +GITEA_SSH_PREFIX = "git@git.add-ideas.de:" +GITEA_HTTPS_PREFIX = "https://git.add-ideas.de/" +GITEA_REPOSITORY_PATH = re.compile(r"add-ideas/[a-z0-9][a-z0-9-]*[.]git") -def parse_args() -> argparse.Namespace: +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Clone GovOPlaN repositories listed in repositories.json.") parser.add_argument("--check", action="store_true", help="Only report missing repositories.") parser.add_argument("--parent", type=Path, help="Override checkout parent directory.") - return parser.parse_args() + parser.add_argument( + "--repo", + action="append", + default=[], + metavar="NAME", + help="Clone only this registered repository; repeat for more than one.", + ) + parser.add_argument( + "--exclude-repo", + action="append", + default=[], + metavar="NAME", + help="Exclude this registered repository; repeat for more than one.", + ) + parser.add_argument( + "--transport", + choices=(REGISTERED_TRANSPORT, PUBLIC_HTTPS_TRANSPORT), + default=REGISTERED_TRANSPORT, + help=( + "Clone registered remotes unchanged, or convert public " + "git.add-ideas.de SSH remotes to anonymous HTTPS." + ), + ) + return parser.parse_args(argv) -def main() -> int: - args = parse_args() - manifest = json.loads((ROOT / "repositories.json").read_text()) +def clone_remote(remote: str, *, transport: str) -> str: + if transport == REGISTERED_TRANSPORT: + return remote + if transport != PUBLIC_HTTPS_TRANSPORT: + raise ValueError(f"unsupported repository transport: {transport}") + if remote.startswith(GITEA_SSH_PREFIX): + repository_path = remote.removeprefix(GITEA_SSH_PREFIX) + else: + parsed = urlsplit(remote) + if ( + parsed.scheme not in {"https", "ssh"} + or parsed.hostname != "git.add-ideas.de" + or parsed.port is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError( + "public HTTPS bootstrap only accepts registered " + "git.add-ideas.de remotes" + ) + if parsed.scheme == "https" and parsed.username is not None: + raise ValueError("public HTTPS bootstrap does not accept credentials") + if parsed.scheme == "ssh" and parsed.username != "git": + raise ValueError("registered Gitea SSH remotes must use the git account") + repository_path = parsed.path.removeprefix("/") + if GITEA_REPOSITORY_PATH.fullmatch(repository_path) is None: + raise ValueError("registered Gitea repository path is malformed") + return GITEA_HTTPS_PREFIX + repository_path + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + manifest = json.loads((ROOT / "repositories.json").read_text(encoding="utf-8")) parent = args.parent or Path(manifest["default_parent"]) - missing: list[dict[str, str]] = [] + missing: list[tuple[dict[str, str], Path, str]] = [] + repositories = manifest["repositories"] + names = [entry["name"] for entry in repositories] + if len(names) != len(set(names)): + raise ValueError("repository registry contains duplicate names") + requested = set(args.repo) + excluded = set(args.exclude_repo) + unknown = (requested | excluded) - set(names) + if unknown: + raise ValueError(f"unknown registered repositories: {', '.join(sorted(unknown))}") + overlap = requested & excluded + if overlap: + raise ValueError( + f"repositories cannot be selected and excluded: {', '.join(sorted(overlap))}" + ) - for entry in manifest["repositories"]: + for entry in repositories: + if (requested and entry["name"] not in requested) or entry["name"] in excluded: + continue repo = parent / entry["path"] if repo.exists(): continue - missing.append(entry) + transport = entry.get("bootstrap_transport", args.transport) + remote = clone_remote(entry["remote"], transport=transport) + missing.append((entry, repo, remote)) + + environment = os.environ.copy() + environment.update( + { + "GIT_ASKPASS": "/bin/false", + "GIT_TERMINAL_PROMPT": "0", + } + ) + for entry, repo, remote in missing: print(f"missing: {entry['name']} -> {repo}") if not args.check: - subprocess.run(["git", "clone", entry["remote"], str(repo)], check=True) + subprocess.run( + [ + "git", + "-c", + "credential.helper=", + "clone", + "--", + remote, + str(repo), + ], + check=True, + env=environment, + ) if args.check: return 1 if missing else 0