446 lines
18 KiB
Python
446 lines
18 KiB
Python
"""Fail-closed provenance checks for catalog source release tags."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import tomllib
|
|
from typing import Iterable
|
|
from urllib.parse import urlsplit
|
|
|
|
from .git_state import (
|
|
collect_repository_snapshot,
|
|
git_text,
|
|
sanitized_git_environment,
|
|
scoped_git_command,
|
|
)
|
|
from .repository_tag import RemoteTagResult, ref_commit, remote_tag_commit
|
|
from .version_alignment import repository_version_issues
|
|
from .workspace import load_repository_specs, resolve_repo_path
|
|
|
|
|
|
_CATALOG_PYTHON_REF = re.compile(
|
|
r"/(?P<repo>govoplan-[a-z0-9-]+)\.git@v(?P<version>[^\s;]+)$"
|
|
)
|
|
_VERSION = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SourceTagProvenanceIssue:
|
|
repo: str
|
|
tag: str
|
|
message: str
|
|
|
|
def describe(self) -> str:
|
|
return f"{self.repo} {self.tag}: {self.message}"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CatalogSourceSelection:
|
|
all_versions: dict[str, str]
|
|
selected_versions: dict[str, str]
|
|
selected_commits: dict[str, str]
|
|
selected_tag_objects: dict[str, str]
|
|
issues: tuple[SourceTagProvenanceIssue, ...] = ()
|
|
|
|
|
|
def catalog_source_selection(payload: object) -> CatalogSourceSelection:
|
|
"""Extract all immutable Python source refs and the explicitly selected units."""
|
|
|
|
if not isinstance(payload, dict):
|
|
return CatalogSourceSelection(
|
|
all_versions={},
|
|
selected_versions={},
|
|
selected_commits={},
|
|
selected_tag_objects={},
|
|
issues=(SourceTagProvenanceIssue("catalog", "", "catalog must be a JSON object"),),
|
|
)
|
|
|
|
versions: dict[str, str] = {}
|
|
issues: list[SourceTagProvenanceIssue] = []
|
|
entries: list[tuple[str, object]] = [("core_release", payload.get("core_release"))]
|
|
modules = payload.get("modules")
|
|
if isinstance(modules, list):
|
|
entries.extend((f"modules[{index}]", entry) for index, entry in enumerate(modules))
|
|
|
|
for source, raw_entry in entries:
|
|
if not isinstance(raw_entry, dict):
|
|
continue
|
|
python_ref = raw_entry.get("python_ref")
|
|
match = _CATALOG_PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None
|
|
if match is None:
|
|
continue # The catalog version-alignment gate reports malformed refs.
|
|
repo = match.group("repo")
|
|
version = match.group("version").removeprefix("v")
|
|
previous = versions.setdefault(repo, version)
|
|
if previous != version:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
f"v{version}",
|
|
f"{source} conflicts with the catalog's earlier v{previous} source ref",
|
|
)
|
|
)
|
|
|
|
release = payload.get("release")
|
|
selected_units = release.get("selected_units") if isinstance(release, dict) else None
|
|
selected: dict[str, str] = {}
|
|
selected_commits: dict[str, str] = {}
|
|
selected_tag_objects: dict[str, str] = {}
|
|
if not isinstance(selected_units, list) or not selected_units:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
"catalog",
|
|
"",
|
|
"release.selected_units is missing or empty; selected source provenance cannot be established",
|
|
)
|
|
)
|
|
else:
|
|
for unit in selected_units:
|
|
if not isinstance(unit, dict):
|
|
continue
|
|
repo = unit.get("repo")
|
|
version = unit.get("version")
|
|
if isinstance(repo, str) and isinstance(version, str):
|
|
selected[repo] = version.removeprefix("v")
|
|
commit = unit.get("commit_sha")
|
|
tag_object = unit.get("tag_object_sha")
|
|
if isinstance(commit, str) and re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", commit):
|
|
selected_commits[repo] = commit.lower()
|
|
else:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(repo, f"v{version.removeprefix('v')}", "selected unit has no valid commit_sha provenance")
|
|
)
|
|
if isinstance(tag_object, str) and re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", tag_object):
|
|
selected_tag_objects[repo] = tag_object.lower()
|
|
else:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(repo, f"v{version.removeprefix('v')}", "selected unit has no valid tag_object_sha provenance")
|
|
)
|
|
|
|
return CatalogSourceSelection(
|
|
all_versions=versions,
|
|
selected_versions=selected,
|
|
selected_commits=selected_commits,
|
|
selected_tag_objects=selected_tag_objects,
|
|
issues=tuple(issues),
|
|
)
|
|
|
|
|
|
def source_tag_provenance_issues(
|
|
*,
|
|
repo_versions: dict[str, str],
|
|
workspace: Path,
|
|
remote: str = "origin",
|
|
require_head_repos: Iterable[str] = (),
|
|
expected_commits: dict[str, str] | None = None,
|
|
expected_tag_objects: dict[str, str] | None = None,
|
|
) -> tuple[SourceTagProvenanceIssue, ...]:
|
|
"""Verify immutable local/remote tags and selected clean source HEADs.
|
|
|
|
Every requested ref must be an annotated tag in the local object store and
|
|
on the configured remote, and both tag objects must peel to the same commit.
|
|
Repositories in ``require_head_repos`` additionally have to be clean,
|
|
version-aligned worktrees whose requested tag peels to the current HEAD.
|
|
"""
|
|
|
|
remote = remote.strip() or "origin"
|
|
require_head = set(require_head_repos)
|
|
expected_commits = expected_commits or {}
|
|
expected_tag_objects = expected_tag_objects or {}
|
|
requested = sorted(repo_versions.items())
|
|
specs = {spec.name: spec for spec in load_repository_specs(include_website=False)}
|
|
issues: list[SourceTagProvenanceIssue] = []
|
|
|
|
for repo, requested_version in requested:
|
|
version = requested_version.removeprefix("v").strip()
|
|
tag = f"v{version}"
|
|
spec = specs.get(repo)
|
|
if spec is None:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "repository is not registered in repositories.json"))
|
|
continue
|
|
if not _VERSION.fullmatch(version):
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "requested version is not a valid release version"))
|
|
continue
|
|
|
|
path = resolve_repo_path(spec, workspace)
|
|
snapshot = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=tag, online=False)
|
|
if not snapshot.exists or not snapshot.is_git:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "local source repository is missing or is not a Git repository"))
|
|
continue
|
|
if snapshot.safe_directory_required:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "Git safe.directory approval is required"))
|
|
continue
|
|
if not snapshot.has_head:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "local source repository has no HEAD commit"))
|
|
continue
|
|
|
|
head_commit = git_text(path, "rev-parse", "HEAD")
|
|
local_commit = ref_commit(path, f"refs/tags/{tag}")
|
|
local_object = git_text(path, "rev-parse", "--verify", f"refs/tags/{tag}") if local_commit else None
|
|
local_type = git_text(path, "cat-file", "-t", f"refs/tags/{tag}") if local_commit else None
|
|
if not local_commit:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "annotated tag is missing from the local source repository"))
|
|
elif local_type != "tag":
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "local release tag is lightweight; an annotated tag is required"))
|
|
|
|
remote_url = git_text(path, "remote", "get-url", remote)
|
|
if remote_url == "":
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, f"configured Git remote {remote!r} does not exist"))
|
|
remote_result = None
|
|
else:
|
|
remote_result = inspect_remote_tag(path=path, remote=remote, remote_url=remote_url, tag=tag)
|
|
if remote_result.error:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, remote_result.error))
|
|
elif not remote_result.commit:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, f"annotated tag is missing from remote {remote!r}"))
|
|
elif not remote_result.annotated:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, f"remote {remote!r} release tag is lightweight; an annotated tag is required"))
|
|
|
|
remote_commit = remote_result.commit if remote_result is not None and not remote_result.error else None
|
|
remote_annotated = remote_result.annotated if remote_result is not None else False
|
|
if local_commit and local_type == "tag" and remote_commit and remote_annotated and local_commit != remote_commit:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"local tag peels to {local_commit[:12]}, but remote {remote!r} peels to {remote_commit[:12]}",
|
|
)
|
|
)
|
|
if (
|
|
local_object
|
|
and local_type == "tag"
|
|
and remote_result is not None
|
|
and remote_result.annotated
|
|
and remote_result.tag_object
|
|
and local_object != remote_result.tag_object
|
|
):
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
"local and remote annotated tag objects differ; the release annotation provenance is inconsistent",
|
|
)
|
|
)
|
|
|
|
expected_commit = expected_commits.get(repo)
|
|
if expected_commit and local_commit and local_commit.lower() != expected_commit.lower():
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"local tag no longer matches signed commit_sha {expected_commit}",
|
|
)
|
|
)
|
|
expected_object = expected_tag_objects.get(repo)
|
|
if expected_object and local_object and local_object.lower() != expected_object.lower():
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"local tag no longer matches signed tag_object_sha {expected_object}",
|
|
)
|
|
)
|
|
|
|
issues.extend(tagged_source_version_issues(path=path, repo=repo, tag=tag, expected_version=version))
|
|
|
|
if repo in require_head:
|
|
if not snapshot.branch:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "selected source HEAD is detached; a named branch is required"))
|
|
if snapshot.dirty_entries:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, "selected source worktree is not clean"))
|
|
if snapshot.behind:
|
|
issues.append(SourceTagProvenanceIssue(repo, tag, f"selected source branch is behind {snapshot.upstream}"))
|
|
for issue in repository_version_issues(path, expected_version=version):
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"selected source version alignment failed: {issue.source}={issue.actual!r}, expected {issue.expected!r}",
|
|
)
|
|
)
|
|
if local_commit and local_commit != head_commit:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"local tag peels to {local_commit[:12]}, not selected HEAD {head_commit[:12]}",
|
|
)
|
|
)
|
|
if remote_commit and remote_commit != head_commit:
|
|
issues.append(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"remote tag peels to {remote_commit[:12]}, not selected HEAD {head_commit[:12]}",
|
|
)
|
|
)
|
|
|
|
return tuple(_deduplicate(issues))
|
|
|
|
|
|
def selected_source_provenance(
|
|
*,
|
|
repo_versions: dict[str, str],
|
|
workspace: Path,
|
|
) -> dict[str, dict[str, str]]:
|
|
"""Return immutable source object identifiers after the provenance gate passes."""
|
|
|
|
specs = {spec.name: spec for spec in load_repository_specs(include_website=False)}
|
|
result: dict[str, dict[str, str]] = {}
|
|
for repo, requested_version in sorted(repo_versions.items()):
|
|
spec = specs[repo]
|
|
path = resolve_repo_path(spec, workspace)
|
|
tag = f"v{requested_version.removeprefix('v')}"
|
|
result[repo] = {
|
|
"commit_sha": git_text(path, "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"),
|
|
"tag_object_sha": git_text(path, "rev-parse", "--verify", f"refs/tags/{tag}"),
|
|
}
|
|
return result
|
|
|
|
|
|
def inspect_remote_tag(*, path: Path, remote: str, remote_url: str, tag: str) -> RemoteTagResult:
|
|
"""Inspect refs over same-host HTTPS when possible, then fall back to the configured remote."""
|
|
|
|
environment = {
|
|
**os.environ,
|
|
"GIT_TERMINAL_PROMPT": "0",
|
|
"GIT_SSH_COMMAND": os.environ.get(
|
|
"GIT_SSH_COMMAND",
|
|
"ssh -o BatchMode=yes -o ConnectTimeout=8",
|
|
),
|
|
}
|
|
inspection_remote = read_only_https_remote(remote_url)
|
|
if inspection_remote:
|
|
result = remote_tag_commit(
|
|
path,
|
|
remote=inspection_remote,
|
|
tag=tag,
|
|
timeout=5,
|
|
env=environment,
|
|
)
|
|
if not result.error:
|
|
return result
|
|
return remote_tag_commit(
|
|
path,
|
|
remote=remote,
|
|
tag=tag,
|
|
timeout=12,
|
|
env=environment,
|
|
)
|
|
|
|
|
|
def read_only_https_remote(remote_url: str) -> str | None:
|
|
"""Derive the equivalent same-host HTTPS Git URL for a conventional SSH remote."""
|
|
|
|
value = remote_url.strip()
|
|
if value.startswith(("https://", "http://")):
|
|
return value
|
|
if value.startswith("ssh://"):
|
|
parsed = urlsplit(value)
|
|
if parsed.hostname and parsed.port in {None, 22} and parsed.path:
|
|
return f"https://{parsed.hostname}{parsed.path}"
|
|
return None
|
|
match = re.fullmatch(r"(?:[^@/:]+@)?([^/:]+):(.+)", value)
|
|
if match:
|
|
return f"https://{match.group(1)}/{match.group(2).lstrip('/')}"
|
|
return None
|
|
|
|
|
|
def tagged_source_version_issues(
|
|
*,
|
|
path: Path,
|
|
repo: str,
|
|
tag: str,
|
|
expected_version: str,
|
|
) -> tuple[SourceTagProvenanceIssue, ...]:
|
|
"""Check version-bearing files from the tag without changing the worktree."""
|
|
|
|
if not ref_commit(path, f"refs/tags/{tag}"):
|
|
return ()
|
|
tree = _git(path, "ls-tree", "-r", "--name-only", tag)
|
|
if tree.returncode != 0:
|
|
return (SourceTagProvenanceIssue(repo, tag, "could not inspect the local tag's source tree"),)
|
|
files = set(tree.stdout.splitlines())
|
|
declarations: list[tuple[str, str | None]] = []
|
|
|
|
pyproject = _git_file(path, tag, "pyproject.toml")
|
|
if pyproject is None:
|
|
return (SourceTagProvenanceIssue(repo, tag, "tagged source has no pyproject.toml package metadata"),)
|
|
try:
|
|
parsed = tomllib.loads(pyproject)
|
|
project = parsed.get("project")
|
|
declarations.append(("pyproject.toml", project.get("version") if isinstance(project, dict) else None))
|
|
except tomllib.TOMLDecodeError:
|
|
declarations.append(("pyproject.toml", None))
|
|
|
|
for name in ("package.json", "webui/package.json", "webui/package.release.json"):
|
|
if name not in files:
|
|
continue
|
|
declarations.append((name, _json_version(_git_file(path, tag, name))))
|
|
|
|
for name in sorted(files):
|
|
if name.startswith("src/") and name.endswith("/backend/manifest.py"):
|
|
text = _git_file(path, tag, name) or ""
|
|
match = re.search(r'(?m)^\s*(?:version|MODULE_VERSION)\s*=\s*["\']([^"\']+)["\']', text)
|
|
if match:
|
|
declarations.append((name, match.group(1)))
|
|
|
|
return tuple(
|
|
SourceTagProvenanceIssue(
|
|
repo,
|
|
tag,
|
|
f"tagged {source} version is {actual!r}, expected {expected_version!r}",
|
|
)
|
|
for source, actual in declarations
|
|
if actual != expected_version
|
|
)
|
|
|
|
|
|
def _json_version(value: str | None) -> str | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
payload = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
version = payload.get("version") if isinstance(payload, dict) else None
|
|
return version if isinstance(version, str) else None
|
|
|
|
|
|
def _git_file(path: Path, tag: str, name: str) -> str | None:
|
|
result = _git(path, "show", f"{tag}:{name}")
|
|
return result.stdout if result.returncode == 0 else None
|
|
|
|
|
|
def _git(path: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
command = scoped_git_command(path, *args)
|
|
try:
|
|
return subprocess.run(
|
|
command,
|
|
cwd=path,
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=30,
|
|
env=sanitized_git_environment(),
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
return subprocess.CompletedProcess(command, 124, exc.stdout or "", exc.stderr or "Git command timed out")
|
|
|
|
|
|
def _deduplicate(issues: list[SourceTagProvenanceIssue]) -> list[SourceTagProvenanceIssue]:
|
|
result: list[SourceTagProvenanceIssue] = []
|
|
seen: set[tuple[str, str, str]] = set()
|
|
for issue in issues:
|
|
key = (issue.repo, issue.tag, issue.message)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
result.append(issue)
|
|
return result
|