Files
govoplan/tools/repo/sync-python-environment.py
Albrecht Degering 1aea3e7c4f
Some checks are pending
Dependency Audit / dependency-audit (push) Waiting to run
Security Audit / security-audit (push) Waiting to run
fix: repair dev modules and scoped release git trust
2026-07-29 20:54:58 +02:00

674 lines
24 KiB
Python

#!/usr/bin/env python3
"""Synchronize a GovOPlaN Python environment when package metadata changes."""
from __future__ import annotations
import argparse
from dataclasses import dataclass
import hashlib
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys
import tomllib
from typing import Any
META_ROOT = Path(__file__).resolve().parents[2]
STAMP_VERSION = 1
@dataclass(frozen=True, slots=True)
class RequirementEntry:
line: str
install_args: tuple[str, ...]
is_local: bool
key: str
target_path: str | None = None
pyproject: str | None = None
def as_dict(self) -> dict[str, object]:
payload: dict[str, object] = {
"line": self.line,
"install_args": list(self.install_args),
"is_local": self.is_local,
"key": self.key,
}
if self.target_path:
payload["target_path"] = self.target_path
if self.pyproject:
payload["pyproject"] = self.pyproject
return payload
@dataclass(frozen=True, slots=True)
class InstallPlan:
mode: str
reason: str
commands: tuple[tuple[str, ...], ...]
warnings: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class LocalInstallationExpectation:
requirement: RequirementEntry
distribution: str
entry_points: tuple[tuple[str, str, str], ...]
@dataclass(frozen=True, slots=True)
class EnvironmentValidation:
stale_requirements: tuple[RequirementEntry, ...] = ()
issues: tuple[str, ...] = ()
@property
def current(self) -> bool:
return not self.stale_requirements
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--requirements",
type=Path,
default=META_ROOT / "requirements-dev.txt",
help="Requirements file to install. Defaults to requirements-dev.txt.",
)
parser.add_argument(
"--python",
default=sys.executable,
help="Python executable to use for pip. Defaults to the current interpreter.",
)
parser.add_argument(
"--stamp",
type=Path,
default=None,
help="Stamp file used to remember the last synchronized metadata digest.",
)
parser.add_argument("--force", action="store_true", help="Run pip even when the environment stamp is current.")
parser.add_argument("--check", action="store_true", help="Exit 1 if synchronization would be required.")
parser.add_argument("--dry-run", action="store_true", help="Print what would run without changing the environment.")
parser.add_argument("--upgrade-pip", action="store_true", help="Upgrade pip before installing requirements.")
args = parser.parse_args()
requirements = args.requirements.expanduser().resolve()
if not requirements.exists():
print(f"Requirements file not found: {requirements}", file=sys.stderr)
return 2
python = normalize_python_command(args.python)
stamp = (args.stamp or default_stamp_path(python, requirements)).expanduser().resolve()
local_requirements = local_requirement_entries(requirements)
fingerprint = build_fingerprint(requirements=requirements, python=python, local_requirements=local_requirements)
previous = load_stamp(stamp)
current = previous.get("digest") == fingerprint["digest"]
environment = validate_local_installations(python, local_requirements)
if current and environment.current and not args.force:
payload = stamp_payload(
requirements=requirements,
python=python,
fingerprint=fingerprint,
entries=parse_requirement_entries(requirements),
)
if stamp_needs_refresh(previous, payload):
if args.dry_run:
print(f"Python environment is current for {requirements.name}. Stamp metadata would be refreshed.")
print("No pip command required; the environment stamp would be refreshed.")
return 0
if args.check:
print(f"Python environment is current for {requirements.name}.")
return 0
write_stamp(stamp, payload)
print(f"Python environment is current for {requirements.name}. Stamp metadata refreshed.")
return 0
print(f"Python environment is current for {requirements.name}.")
return 0
if current and not environment.current:
print(f"Python environment metadata is current for {requirements.name}, but installed packages need repair.")
else:
print(f"Python environment metadata is stale for {requirements.name}.")
print(f"Tracked inputs: {len(fingerprint['inputs'])}")
for issue in environment.issues:
print(f"warning: {issue}", file=sys.stderr)
if args.check:
return 1
if current and environment.stale_requirements and not args.force:
plan = build_environment_repair_plan(
python=python,
stale_requirements=environment.stale_requirements,
)
else:
plan = build_install_plan(
previous=previous,
fingerprint=fingerprint,
requirements=requirements,
python=python,
local_requirements=local_requirements,
force=args.force,
)
print(f"{plan.mode}: {plan.reason}")
for warning in plan.warnings:
print(f"warning: {warning}", file=sys.stderr)
commands: list[tuple[str, ...]] = []
if args.upgrade_pip:
commands.append((python, "-m", "pip", "install", "--upgrade", "pip"))
commands.extend(plan.commands)
if args.dry_run:
for command in commands:
print("+ " + format_command(command))
if not commands:
print("No pip command required; the environment stamp would be refreshed.")
return 0
for command in commands:
subprocess.run(command, check=True)
repaired_environment = validate_local_installations(python, local_requirements)
if not repaired_environment.current:
for issue in repaired_environment.issues:
print(f"error: {issue}", file=sys.stderr)
print("Python environment synchronization did not install all local package metadata.", file=sys.stderr)
return 2
write_stamp(
stamp,
stamp_payload(
requirements=requirements,
python=python,
fingerprint=fingerprint,
entries=parse_requirement_entries(requirements),
),
)
print(f"Python environment synchronized. Stamp written to {stamp}.")
return 0
def default_stamp_path(python: str, requirements: Path) -> Path:
executable = Path(python)
if executable.name.startswith("python") and executable.parent.name == "bin":
return executable.parent.parent / f".govoplan-sync-{requirements.stem}.json"
return META_ROOT / "runtime" / f".govoplan-sync-{requirements.stem}.json"
def normalize_python_command(value: str) -> str:
if os.sep not in value:
return value
path = Path(value).expanduser()
if not path.is_absolute():
path = Path.cwd() / path
return str(path)
def stamp_payload(
*,
requirements: Path,
python: str,
fingerprint: dict[str, Any],
entries: tuple[RequirementEntry, ...],
) -> dict[str, object]:
return {
"version": STAMP_VERSION,
"digest": fingerprint["digest"],
"requirements": str(requirements),
"python": python,
"inputs": fingerprint["inputs"],
"requirements_entries": [entry.as_dict() for entry in entries],
}
def stamp_needs_refresh(previous: dict[str, Any], payload: dict[str, object]) -> bool:
return any(previous.get(key) != value for key, value in payload.items())
def build_fingerprint(*, requirements: Path, python: str, local_requirements: tuple[RequirementEntry, ...]) -> dict[str, Any]:
hasher = hashlib.sha256()
inputs: list[dict[str, str]] = []
add_text(hasher, "stamp-version", str(STAMP_VERSION))
add_text(hasher, "python", python)
add_file(hasher, inputs, requirements)
for project in local_pyprojects(local_requirements):
add_file(hasher, inputs, project)
return {"digest": hasher.hexdigest(), "inputs": inputs}
def build_install_plan(
*,
previous: dict[str, Any],
fingerprint: dict[str, Any],
requirements: Path,
python: str,
local_requirements: tuple[RequirementEntry, ...],
force: bool,
) -> InstallPlan:
full_command = (python, "-m", "pip", "install", "-r", str(requirements))
if force:
return InstallPlan("Full Python environment sync", "--force was requested.", (full_command,))
if previous.get("version") != STAMP_VERSION:
return InstallPlan("Full Python environment sync", "the environment stamp is missing or uses an older format.", (full_command,))
if previous.get("python") != python:
return InstallPlan("Full Python environment sync", "the Python executable changed.", (full_command,))
previous_inputs = inputs_by_path(previous.get("inputs"))
current_inputs = inputs_by_path(fingerprint.get("inputs"))
if not previous_inputs:
return InstallPlan("Full Python environment sync", "there is no previous input fingerprint.", (full_command,))
requirements_key = str(requirements)
changed_paths = {
path
for path, digest in current_inputs.items()
if previous_inputs.get(path) != digest
}
removed_paths = set(previous_inputs) - set(current_inputs)
stale_requirements = requirements_key in changed_paths or requirements_key in removed_paths
installs: list[tuple[str, ...]] = []
warnings: list[str] = []
if stale_requirements:
delta = requirement_delta_plan(previous.get("requirements_entries"), parse_requirement_entries(requirements))
if delta is None:
return InstallPlan(
"Full Python environment sync",
"requirements changed in a way that needs the full resolver.",
(full_command,),
)
installs.extend(delta.commands)
warnings.extend(delta.warnings)
changed_paths.discard(requirements_key)
removed_paths.discard(requirements_key)
local_by_pyproject = {
entry.pyproject: entry
for entry in local_requirements
if entry.pyproject
}
for path in sorted(changed_paths):
entry = local_by_pyproject.get(path)
if entry is None:
return InstallPlan(
"Full Python environment sync",
f"tracked input changed outside a known local project: {path}",
(full_command,),
)
installs.append(entry.install_args)
if removed_paths:
return InstallPlan(
"Full Python environment sync",
"tracked input files disappeared; using the full requirements installer.",
(full_command,),
)
deduped_installs = tuple(dict.fromkeys(installs))
if deduped_installs:
command = (
python,
"-m",
"pip",
"install",
*(argument for install_args in deduped_installs for argument in install_args),
)
return InstallPlan(
"Selective Python environment sync",
f"installing {len(deduped_installs)} stale local requirement(s) in one resolver transaction.",
(command,),
tuple(warnings),
)
return InstallPlan(
"Stamp-only Python environment sync",
"requirements metadata changed without a pip-installable requirement change.",
(),
tuple(warnings),
)
def build_environment_repair_plan(
*,
python: str,
stale_requirements: tuple[RequirementEntry, ...],
) -> InstallPlan:
command = (
python,
"-m",
"pip",
"install",
*(
argument
for requirement in stale_requirements
for argument in requirement.install_args
),
)
return InstallPlan(
"Selective Python environment repair",
f"reinstalling {len(stale_requirements)} missing or stale local distribution(s).",
(command,),
)
def validate_local_installations(
python: str,
local_requirements: tuple[RequirementEntry, ...],
) -> EnvironmentValidation:
expectations = local_installation_expectations(local_requirements)
if not expectations:
return EnvironmentValidation()
probe_input = [
{
"key": expectation.requirement.key,
"distribution": expectation.distribution,
"entry_points": [
{"group": group, "name": name, "value": value}
for group, name, value in expectation.entry_points
],
}
for expectation in expectations
]
try:
result = subprocess.run(
(python, "-c", _LOCAL_INSTALLATION_PROBE),
check=False,
text=True,
input=json.dumps(probe_input),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=20,
)
except (OSError, subprocess.TimeoutExpired) as exc:
return EnvironmentValidation(
stale_requirements=tuple(item.requirement for item in expectations),
issues=(f"could not inspect installed local distributions ({type(exc).__name__})",),
)
if result.returncode != 0:
detail = result.stderr.strip() or result.stdout.strip() or "installation probe failed"
return EnvironmentValidation(
stale_requirements=tuple(item.requirement for item in expectations),
issues=(f"could not inspect installed local distributions: {detail}",),
)
try:
payload = json.loads(result.stdout)
except json.JSONDecodeError:
return EnvironmentValidation(
stale_requirements=tuple(item.requirement for item in expectations),
issues=("installed local distribution probe returned invalid JSON",),
)
raw_issues = payload.get("issues") if isinstance(payload, dict) else None
issues_by_key = raw_issues if isinstance(raw_issues, dict) else {}
stale = tuple(
expectation.requirement
for expectation in expectations
if expectation.requirement.key in issues_by_key
)
issues = tuple(
f"{expectation.distribution}: {issues_by_key[expectation.requirement.key]}"
for expectation in expectations
if expectation.requirement.key in issues_by_key
)
return EnvironmentValidation(stale_requirements=stale, issues=issues)
def local_installation_expectations(
local_requirements: tuple[RequirementEntry, ...],
) -> tuple[LocalInstallationExpectation, ...]:
expectations: list[LocalInstallationExpectation] = []
for requirement in local_requirements:
if requirement.pyproject is None:
continue
with Path(requirement.pyproject).open("rb") as handle:
payload = tomllib.load(handle)
project = payload.get("project")
if not isinstance(project, dict):
continue
distribution = project.get("name")
if not isinstance(distribution, str) or not distribution.strip():
continue
declared_entry_points: list[tuple[str, str, str]] = []
entry_point_groups = project.get("entry-points")
if isinstance(entry_point_groups, dict):
for group, entries in entry_point_groups.items():
if not isinstance(group, str) or not isinstance(entries, dict):
continue
for name, value in entries.items():
if isinstance(name, str) and isinstance(value, str):
declared_entry_points.append((group, name, value))
expectations.append(
LocalInstallationExpectation(
requirement=requirement,
distribution=distribution,
entry_points=tuple(sorted(declared_entry_points)),
)
)
return tuple(expectations)
def requirement_delta_plan(previous_entries: object, current_entries: tuple[RequirementEntry, ...]) -> InstallPlan | None:
parsed_previous = previous_requirement_entries(previous_entries)
if parsed_previous is None:
return None
previous_by_key = {entry.key: entry for entry in parsed_previous}
current_by_key = {entry.key: entry for entry in current_entries}
changed_current = [
entry
for key, entry in current_by_key.items()
if previous_by_key.get(key) is None or previous_by_key[key].install_args != entry.install_args or previous_by_key[key].line != entry.line
]
removed_keys = set(previous_by_key) - set(current_by_key)
if any(not entry.is_local or not entry.pyproject for entry in changed_current):
return None
warnings = []
if removed_keys:
warnings.append("Removed requirements are not uninstalled automatically; pip install -r had the same limitation.")
commands = tuple((entry.install_args) for entry in changed_current)
return InstallPlan(
"Requirement delta",
"requirements file changed only in local editable requirements.",
commands,
tuple(warnings),
)
def previous_requirement_entries(value: object) -> tuple[RequirementEntry, ...] | None:
if not isinstance(value, list):
return None
entries: list[RequirementEntry] = []
for item in value:
if not isinstance(item, dict):
return None
raw_install_args = item.get("install_args")
if not isinstance(raw_install_args, list) or not all(isinstance(arg, str) for arg in raw_install_args):
return None
line = item.get("line")
key = item.get("key")
if not isinstance(line, str) or not isinstance(key, str):
return None
entries.append(
RequirementEntry(
line=line,
install_args=tuple(raw_install_args),
is_local=bool(item.get("is_local")),
key=key,
target_path=item.get("target_path") if isinstance(item.get("target_path"), str) else None,
pyproject=item.get("pyproject") if isinstance(item.get("pyproject"), str) else None,
)
)
return tuple(entries)
def inputs_by_path(value: object) -> dict[str, str]:
if not isinstance(value, list):
return {}
inputs: dict[str, str] = {}
for item in value:
if not isinstance(item, dict):
continue
path = item.get("path")
digest = item.get("sha256")
if isinstance(path, str) and isinstance(digest, str):
inputs[path] = digest
return inputs
def local_requirement_entries(requirements: Path) -> tuple[RequirementEntry, ...]:
return tuple(entry for entry in parse_requirement_entries(requirements) if entry.is_local and entry.pyproject)
def parse_requirement_entries(requirements: Path) -> tuple[RequirementEntry, ...]:
entries: list[RequirementEntry] = []
for raw_line in requirements.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or line.startswith(("-r ", "--requirement")):
continue
editable = editable_target(line)
target = editable or local_path_target(line)
if target is None:
entries.append(RequirementEntry(line=line, install_args=(line,), is_local=False, key=f"requirement:{line}"))
continue
path = resolve_requirement_path(requirements.parent, target)
if path is None:
entries.append(RequirementEntry(line=line, install_args=(line,), is_local=False, key=f"requirement:{line}"))
continue
pyproject = path / "pyproject.toml"
resolved_pyproject = str(pyproject.resolve()) if pyproject.exists() else None
install_target = absolute_local_target(requirements.parent, target)
install_args = ("-e", install_target) if editable is not None else (install_target,)
key = f"local:{resolved_pyproject or path.resolve()}"
entries.append(
RequirementEntry(
line=line,
install_args=install_args,
is_local=True,
key=key,
target_path=str(path.resolve()),
pyproject=resolved_pyproject,
)
)
return tuple(entries)
def local_pyprojects(local_requirements: tuple[RequirementEntry, ...]) -> tuple[Path, ...]:
projects = {Path(entry.pyproject) for entry in local_requirements if entry.pyproject}
return tuple(sorted(projects))
def editable_target(line: str) -> str | None:
if line.startswith("-e "):
return line[3:].strip()
if line.startswith("--editable "):
return line.removeprefix("--editable ").strip()
if line.startswith("--editable="):
return line.removeprefix("--editable=").strip()
return None
def local_path_target(line: str) -> str | None:
if line.startswith(("./", "../", "/")):
return line
return None
def resolve_requirement_path(base: Path, target: str) -> Path | None:
cleaned, _suffix = split_local_target(target)
if "://" in cleaned or cleaned.startswith("git+"):
return None
if not cleaned:
return None
return (base / cleaned).resolve()
def absolute_local_target(base: Path, target: str) -> str:
path_part, suffix = split_local_target(target)
return f"{(base / path_part).resolve()}{suffix}"
def split_local_target(target: str) -> tuple[str, str]:
cleaned = target.split("#", 1)[0].strip()
fragment = target[len(cleaned) :].strip()
extras = ""
if "[" in cleaned:
cleaned, extra_part = cleaned.split("[", 1)
extras = f"[{extra_part}"
return cleaned.strip(), f"{extras}{fragment}"
def add_file(hasher: hashlib._Hash, inputs: list[dict[str, str]], path: Path) -> None:
data = path.read_bytes()
digest = hashlib.sha256(data).hexdigest()
add_text(hasher, str(path), digest)
inputs.append({"path": str(path), "sha256": digest})
def add_text(hasher: hashlib._Hash, key: str, value: str) -> None:
hasher.update(key.encode("utf-8"))
hasher.update(b"\0")
hasher.update(value.encode("utf-8"))
hasher.update(b"\0")
def load_stamp(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
return payload if isinstance(payload, dict) else {}
def write_stamp(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def format_command(command: tuple[str, ...]) -> str:
return " ".join(shlex.quote(item) for item in command)
_LOCAL_INSTALLATION_PROBE = """
import importlib.metadata
import json
import sys
expectations = json.load(sys.stdin)
issues = {}
for expectation in expectations:
key = expectation["key"]
distribution_name = expectation["distribution"]
try:
distribution = importlib.metadata.distribution(distribution_name)
except importlib.metadata.PackageNotFoundError:
issues[key] = "distribution is not installed"
continue
installed = {
(entry_point.group, entry_point.name): entry_point.value
for entry_point in distribution.entry_points
}
mismatches = []
for entry_point in expectation["entry_points"]:
identity = (entry_point["group"], entry_point["name"])
if installed.get(identity) != entry_point["value"]:
mismatches.append(f"{identity[0]}:{identity[1]}")
if mismatches:
issues[key] = "entry-point metadata is missing or stale: " + ", ".join(mismatches)
print(json.dumps({"issues": issues}, sort_keys=True))
"""
if __name__ == "__main__":
raise SystemExit(main())