ci: harden repository bootstrap transport

repositories.json

@@ -64,7 +64,7 @@
     {"name": "govoplan-templates", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-templates.git", "path": "govoplan-templates"},
     {"name": "govoplan-tenancy", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-tenancy.git", "path": "govoplan-tenancy"},
     {"name": "govoplan-transparency", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-transparency.git", "path": "govoplan-transparency"},
-    {"name": "addideas-govoplan-website", "category": "website", "subtype": "public-site", "remote": "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", "path": "addideas-govoplan-website"},
+    {"name": "addideas-govoplan-website", "category": "website", "subtype": "public-site", "remote": "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", "path": "addideas-govoplan-website", "bootstrap_transport": "registered"},
     {"name": "govoplan-workflow", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-workflow.git", "path": "govoplan-workflow"},
     {"name": "govoplan-xoev", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xoev.git", "path": "govoplan-xoev"},
     {"name": "govoplan-xrechnung", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xrechnung.git", "path": "govoplan-xrechnung"},

tests/test_repository_bootstrap.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+META_ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = META_ROOT / "tools" / "repo" / "bootstrap-repositories.py"
+
+
+def load_bootstrap_module():
+    spec = importlib.util.spec_from_file_location("bootstrap_repositories", SCRIPT)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Could not load {SCRIPT}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+class RepositoryBootstrapTests(unittest.TestCase):
+    def test_public_https_transport_rewrites_registered_gitea_remotes(self) -> None:
+        bootstrap = load_bootstrap_module()
+
+        self.assertEqual(
+            "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+            bootstrap.clone_remote(
+                "git@git.add-ideas.de:add-ideas/govoplan-core.git",
+                transport=bootstrap.PUBLIC_HTTPS_TRANSPORT,
+            ),
+        )
+        self.assertEqual(
+            "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+            bootstrap.clone_remote(
+                "ssh://git@git.add-ideas.de/add-ideas/govoplan-core.git",
+                transport=bootstrap.PUBLIC_HTTPS_TRANSPORT,
+            ),
+        )
+        self.assertEqual(
+            "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+            bootstrap.clone_remote(
+                "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+                transport=bootstrap.PUBLIC_HTTPS_TRANSPORT,
+            ),
+        )
+
+    def test_registered_transport_preserves_the_manifest_remote(self) -> None:
+        bootstrap = load_bootstrap_module()
+        remote = "git@example.test:private/repository.git"
+
+        self.assertEqual(
+            remote,
+            bootstrap.clone_remote(
+                remote,
+                transport=bootstrap.REGISTERED_TRANSPORT,
+            ),
+        )
+
+    def test_public_https_transport_fails_closed_for_other_hosts(self) -> None:
+        bootstrap = load_bootstrap_module()
+
+        unsafe_remotes = (
+            "git@example.test:add-ideas/govoplan-core.git",
+            "https://token@git.add-ideas.de/add-ideas/govoplan-core.git",
+            "https://git.add-ideas.de:443/add-ideas/govoplan-core.git",
+            "https://git.add-ideas.de/add-ideas/../govoplan-core.git",
+            "https://git.add-ideas.de/add-ideas/govoplan-core.git?ref=main",
+            "ssh://root@git.add-ideas.de/add-ideas/govoplan-core.git",
+        )
+        for remote in unsafe_remotes:
+            with self.subTest(remote=remote), self.assertRaises(ValueError):
+                bootstrap.clone_remote(
+                    remote,
+                    transport=bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                )
+
+    def test_main_clones_missing_repositories_over_public_https(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": "govoplan-core",
+                                "path": "govoplan-core",
+                                "remote": (
+                                    "git@git.add-ideas.de:"
+                                    "add-ideas/govoplan-core.git"
+                                ),
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with (
+                patch.object(bootstrap, "ROOT", root),
+                patch.object(bootstrap.subprocess, "run") as runner,
+            ):
+                status = bootstrap.main(
+                    [
+                        "--parent",
+                        str(parent),
+                        "--transport",
+                        bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                    ]
+                )
+
+        self.assertEqual(0, status)
+        runner.assert_called_once_with(
+            [
+                "git",
+                "-c",
+                "credential.helper=",
+                "clone",
+                "--",
+                "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+                str(parent / "govoplan-core"),
+            ],
+            check=True,
+            env=runner.call_args.kwargs["env"],
+        )
+        environment = runner.call_args.kwargs["env"]
+        self.assertEqual("/bin/false", environment["GIT_ASKPASS"])
+        self.assertEqual("0", environment["GIT_TERMINAL_PROMPT"])
+
+    def test_main_validates_every_remote_before_cloning(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": "govoplan-core",
+                                "path": "govoplan-core",
+                                "remote": (
+                                    "git@git.add-ideas.de:"
+                                    "add-ideas/govoplan-core.git"
+                                ),
+                            },
+                            {
+                                "name": "unsafe",
+                                "path": "unsafe",
+                                "remote": "git@example.test:private/unsafe.git",
+                            },
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with (
+                patch.object(bootstrap, "ROOT", root),
+                patch.object(bootstrap.subprocess, "run") as runner,
+                self.assertRaises(ValueError),
+            ):
+                bootstrap.main(
+                    [
+                        "--parent",
+                        str(parent),
+                        "--transport",
+                        bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                    ]
+                )
+
+        runner.assert_not_called()
+
+    def test_main_preserves_an_explicit_private_repository_transport(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": "website",
+                                "path": "website",
+                                "remote": (
+                                    "git@git.add-ideas.de:"
+                                    "add-ideas/addideas-govoplan-website.git"
+                                ),
+                                "bootstrap_transport": (
+                                    bootstrap.REGISTERED_TRANSPORT
+                                ),
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with (
+                patch.object(bootstrap, "ROOT", root),
+                patch.object(bootstrap.subprocess, "run") as runner,
+            ):
+                status = bootstrap.main(
+                    [
+                        "--parent",
+                        str(parent),
+                        "--transport",
+                        bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                    ]
+                )
+
+        self.assertEqual(0, status)
+        command = runner.call_args.args[0]
+        self.assertEqual(
+            "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git",
+            command[-2],
+        )
+
+    def test_main_limits_bootstrap_to_selected_repositories(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": name,
+                                "path": name,
+                                "remote": (
+                                    "git@git.add-ideas.de:add-ideas/"
+                                    f"{name}.git"
+                                ),
+                            }
+                            for name in ("govoplan-core", "govoplan-poll")
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            for repository_filter in (
+                ["--repo", "govoplan-core"],
+                ["--exclude-repo", "govoplan-poll"],
+            ):
+                with (
+                    self.subTest(repository_filter=repository_filter),
+                    patch.object(bootstrap, "ROOT", root),
+                    patch.object(bootstrap.subprocess, "run") as runner,
+                ):
+                    status = bootstrap.main(
+                        [
+                            "--parent",
+                            str(parent),
+                            "--transport",
+                            bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                            *repository_filter,
+                        ]
+                    )
+                    self.assertEqual(0, status)
+                    runner.assert_called_once()
+                    self.assertEqual(
+                        "https://git.add-ideas.de/add-ideas/govoplan-core.git",
+                        runner.call_args.args[0][-2],
+                    )
+
+    def test_main_rejects_unknown_or_conflicting_repository_filters(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": "govoplan-core",
+                                "path": "govoplan-core",
+                                "remote": (
+                                    "git@git.add-ideas.de:"
+                                    "add-ideas/govoplan-core.git"
+                                ),
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with (
+                patch.object(bootstrap, "ROOT", root),
+                patch.object(bootstrap.subprocess, "run") as runner,
+            ):
+                with self.assertRaisesRegex(ValueError, "unknown registered"):
+                    bootstrap.main(["--repo", "govoplan-missing"])
+                with self.assertRaisesRegex(ValueError, "selected and excluded"):
+                    bootstrap.main(
+                        [
+                            "--repo",
+                            "govoplan-core",
+                            "--exclude-repo",
+                            "govoplan-core",
+                        ]
+                    )
+
+        runner.assert_not_called()
+
+    def test_check_reports_missing_without_cloning(self) -> None:
+        bootstrap = load_bootstrap_module()
+        with tempfile.TemporaryDirectory(prefix="govoplan-bootstrap-") as directory:
+            root = Path(directory)
+            parent = root / "checkouts"
+            parent.mkdir()
+            root.joinpath("repositories.json").write_text(
+                json.dumps(
+                    {
+                        "default_parent": str(parent),
+                        "repositories": [
+                            {
+                                "name": "govoplan-core",
+                                "path": "govoplan-core",
+                                "remote": (
+                                    "git@git.add-ideas.de:"
+                                    "add-ideas/govoplan-core.git"
+                                ),
+                            }
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with (
+                patch.object(bootstrap, "ROOT", root),
+                patch.object(bootstrap.subprocess, "run") as runner,
+            ):
+                status = bootstrap.main(
+                    [
+                        "--check",
+                        "--parent",
+                        str(parent),
+                        "--transport",
+                        bootstrap.PUBLIC_HTTPS_TRANSPORT,
+                    ]
+                )
+
+        self.assertEqual(1, status)
+        runner.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/repo/bootstrap-repositories.py

@@ -3,35 +3,136 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
+import re
 import subprocess
 import sys
 from pathlib import Path
+from urllib.parse import urlsplit
 
 
 ROOT = Path(__file__).resolve().parents[2]
+REGISTERED_TRANSPORT = "registered"
+PUBLIC_HTTPS_TRANSPORT = "public-https"
+GITEA_SSH_PREFIX = "git@git.add-ideas.de:"
+GITEA_HTTPS_PREFIX = "https://git.add-ideas.de/"
+GITEA_REPOSITORY_PATH = re.compile(r"add-ideas/[a-z0-9][a-z0-9-]*[.]git")
 
 
-def parse_args() -> argparse.Namespace:
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Clone GovOPlaN repositories listed in repositories.json.")
     parser.add_argument("--check", action="store_true", help="Only report missing repositories.")
     parser.add_argument("--parent", type=Path, help="Override checkout parent directory.")
-    return parser.parse_args()
+    parser.add_argument(
+        "--repo",
+        action="append",
+        default=[],
+        metavar="NAME",
+        help="Clone only this registered repository; repeat for more than one.",
+    )
+    parser.add_argument(
+        "--exclude-repo",
+        action="append",
+        default=[],
+        metavar="NAME",
+        help="Exclude this registered repository; repeat for more than one.",
+    )
+    parser.add_argument(
+        "--transport",
+        choices=(REGISTERED_TRANSPORT, PUBLIC_HTTPS_TRANSPORT),
+        default=REGISTERED_TRANSPORT,
+        help=(
+            "Clone registered remotes unchanged, or convert public "
+            "git.add-ideas.de SSH remotes to anonymous HTTPS."
+        ),
+    )
+    return parser.parse_args(argv)
 
 
-def main() -> int:
+def clone_remote(remote: str, *, transport: str) -> str:
-    args = parse_args()
+    if transport == REGISTERED_TRANSPORT:
-    manifest = json.loads((ROOT / "repositories.json").read_text())
+        return remote
+    if transport != PUBLIC_HTTPS_TRANSPORT:
+        raise ValueError(f"unsupported repository transport: {transport}")
+    if remote.startswith(GITEA_SSH_PREFIX):
+        repository_path = remote.removeprefix(GITEA_SSH_PREFIX)
+    else:
+        parsed = urlsplit(remote)
+        if (
+            parsed.scheme not in {"https", "ssh"}
+            or parsed.hostname != "git.add-ideas.de"
+            or parsed.port is not None
+            or parsed.password is not None
+            or parsed.query
+            or parsed.fragment
+        ):
+            raise ValueError(
+                "public HTTPS bootstrap only accepts registered "
+                "git.add-ideas.de remotes"
+            )
+        if parsed.scheme == "https" and parsed.username is not None:
+            raise ValueError("public HTTPS bootstrap does not accept credentials")
+        if parsed.scheme == "ssh" and parsed.username != "git":
+            raise ValueError("registered Gitea SSH remotes must use the git account")
+        repository_path = parsed.path.removeprefix("/")
+    if GITEA_REPOSITORY_PATH.fullmatch(repository_path) is None:
+        raise ValueError("registered Gitea repository path is malformed")
+    return GITEA_HTTPS_PREFIX + repository_path
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    manifest = json.loads((ROOT / "repositories.json").read_text(encoding="utf-8"))
     parent = args.parent or Path(manifest["default_parent"])
-    missing: list[dict[str, str]] = []
+    missing: list[tuple[dict[str, str], Path, str]] = []
+    repositories = manifest["repositories"]
+    names = [entry["name"] for entry in repositories]
+    if len(names) != len(set(names)):
+        raise ValueError("repository registry contains duplicate names")
+    requested = set(args.repo)
+    excluded = set(args.exclude_repo)
+    unknown = (requested | excluded) - set(names)
+    if unknown:
+        raise ValueError(f"unknown registered repositories: {', '.join(sorted(unknown))}")
+    overlap = requested & excluded
+    if overlap:
+        raise ValueError(
+            f"repositories cannot be selected and excluded: {', '.join(sorted(overlap))}"
+        )
 
-    for entry in manifest["repositories"]:
+    for entry in repositories:
+        if (requested and entry["name"] not in requested) or entry["name"] in excluded:
+            continue
         repo = parent / entry["path"]
         if repo.exists():
             continue
-        missing.append(entry)
+        transport = entry.get("bootstrap_transport", args.transport)
+        remote = clone_remote(entry["remote"], transport=transport)
+        missing.append((entry, repo, remote))
+
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "GIT_ASKPASS": "/bin/false",
+            "GIT_TERMINAL_PROMPT": "0",
+        }
+    )
+    for entry, repo, remote in missing:
         print(f"missing: {entry['name']} -> {repo}")
         if not args.check:
-            subprocess.run(["git", "clone", entry["remote"], str(repo)], check=True)
+            subprocess.run(
+                [
+                    "git",
+                    "-c",
+                    "credential.helper=",
+                    "clone",
+                    "--",
+                    remote,
+                    str(repo),
+                ],
+                check=True,
+                env=environment,
+            )
 
     if args.check:
         return 1 if missing else 0