Harden source release preparation and record verified security follow-up
This commit is contained in:
@@ -0,0 +1,380 @@
|
||||
"""Receipt-bound, out-of-run preparation of the real developer meta-package.
|
||||
|
||||
This deliberately does not commit, tag, publish, or update a running release
|
||||
console. The complete generated file is reviewed in a separate source checkout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
import stat
|
||||
import tempfile
|
||||
import tomllib
|
||||
|
||||
from .git_state import (
|
||||
collect_repository_snapshot,
|
||||
git,
|
||||
git_text,
|
||||
registered_developer_meta_path,
|
||||
)
|
||||
from .repository_tag import normalize_version, remote_tag_commit, run
|
||||
from .source_provenance import registered_source_origin_issues
|
||||
from .source_tag_batch import (
|
||||
_OBJECT,
|
||||
_owned_path,
|
||||
_source_filesystem,
|
||||
_tracked_worktree,
|
||||
_trusted_ancestry,
|
||||
)
|
||||
from .workspace import META_ROOT, load_repository_specs, resolve_repo_path
|
||||
|
||||
PACKAGE = "packages/govoplan-meta/pyproject.toml"
|
||||
MAX_INPUT_FILES = 128
|
||||
MAX_INPUT_BYTES = 2 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 16 * 1024 * 1024
|
||||
GENERATOR = ".operator/generate-developer-meta-package.py"
|
||||
|
||||
|
||||
class MetaPreparationError(ValueError):
|
||||
"""Preparation is blocked, or an applied file needs explicit reconciliation."""
|
||||
|
||||
|
||||
class MetaPreparationAmbiguous(MetaPreparationError):
|
||||
"""The file effect may have happened and requires explicit reconciliation."""
|
||||
|
||||
|
||||
def preparation_command(*, workspace: Path, target_version: str) -> str:
|
||||
import shlex
|
||||
|
||||
return " ".join(
|
||||
shlex.quote(value)
|
||||
for value in (
|
||||
"python",
|
||||
str(META_ROOT / "tools/release/prepare-developer-meta-package.py"),
|
||||
"--workspace",
|
||||
str(workspace),
|
||||
"--target-version",
|
||||
target_version,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _read_input(path: Path) -> tuple[bytes, dict]:
|
||||
def identity(value):
|
||||
return (
|
||||
value.st_dev,
|
||||
value.st_ino,
|
||||
value.st_uid,
|
||||
value.st_gid,
|
||||
value.st_mode,
|
||||
value.st_size,
|
||||
value.st_mtime_ns,
|
||||
value.st_ctime_ns,
|
||||
)
|
||||
|
||||
before = path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_uid != os.geteuid()
|
||||
or before.st_mode & 0o022
|
||||
or not 0 < before.st_size <= MAX_INPUT_BYTES
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Preparation inputs must be owned, bounded regular files."
|
||||
)
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
with os.fdopen(descriptor, "rb") as source:
|
||||
opened = os.fstat(source.fileno())
|
||||
if identity(opened) != identity(before):
|
||||
raise MetaPreparationError("Preparation input changed before reading.")
|
||||
payload = source.read(before.st_size + 1)
|
||||
if (
|
||||
identity(os.fstat(source.fileno())) != identity(opened)
|
||||
or len(payload) != before.st_size
|
||||
):
|
||||
raise MetaPreparationError("Preparation input changed while reading.")
|
||||
return payload, {
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"identity": [
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_uid,
|
||||
before.st_gid,
|
||||
stat.S_IMODE(before.st_mode),
|
||||
before.st_size,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _snapshot(*, repo_path: Path, target_version: str, output_dirty: bool = False):
|
||||
workspace = repo_path.parent
|
||||
specs = {spec.name: spec for spec in load_repository_specs(include_website=False)}
|
||||
if registered_developer_meta_path(repo_path) != repo_path / PACKAGE:
|
||||
raise MetaPreparationError(
|
||||
"Only the registered Meta nested-package identity can be prepared."
|
||||
)
|
||||
if repo_path.resolve() == META_ROOT.resolve():
|
||||
raise MetaPreparationError(
|
||||
"Prepare a separate registered source checkout, never the running operator tooling."
|
||||
)
|
||||
paths = [repo_path / PACKAGE, repo_path / "requirements-release.txt"]
|
||||
for path in workspace.glob("govoplan-*/pyproject.toml"):
|
||||
paths.append(path)
|
||||
if len(paths) > MAX_INPUT_FILES:
|
||||
raise MetaPreparationError(
|
||||
"Developer composition exceeds its input-file bound."
|
||||
)
|
||||
if workspace / "govoplan-core/pyproject.toml" not in paths:
|
||||
raise MetaPreparationError(
|
||||
"Prepare and commit matching Core metadata before Meta preparation."
|
||||
)
|
||||
repositories = {"govoplan": repo_path}
|
||||
for path in paths[2:]:
|
||||
name = path.parent.name
|
||||
if (
|
||||
name not in specs
|
||||
or resolve_repo_path(specs[name], workspace) != path.parent
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Developer composition contains an unregistered package checkout."
|
||||
)
|
||||
repositories[name] = path.parent
|
||||
filesystems = {}
|
||||
for name, path in repositories.items():
|
||||
filesystems[name] = _source_filesystem(path=path, workspace=workspace)
|
||||
_tracked_worktree(path)
|
||||
issues = registered_source_origin_issues(
|
||||
repo_versions={name: target_version for name in repositories},
|
||||
workspace=workspace,
|
||||
remote="origin",
|
||||
)
|
||||
if issues:
|
||||
raise MetaPreparationError(
|
||||
"Preparation source origins do not match the registered repositories."
|
||||
)
|
||||
sources = {}
|
||||
for name, path in sorted(repositories.items()):
|
||||
snapshot = collect_repository_snapshot(
|
||||
specs[name],
|
||||
workspace_root=workspace,
|
||||
target_tag=None,
|
||||
online=False,
|
||||
)
|
||||
dirty_allowed = (
|
||||
output_dirty
|
||||
and name == "govoplan"
|
||||
and snapshot.dirty_entries == (f" M {PACKAGE}",)
|
||||
)
|
||||
if (
|
||||
snapshot.errors
|
||||
or not snapshot.has_head
|
||||
or snapshot.branch != "main"
|
||||
or snapshot.upstream != "origin/main"
|
||||
or snapshot.behind
|
||||
or (snapshot.dirty and not dirty_allowed)
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Preparation requires reviewed clean main sources tracking origin/main."
|
||||
)
|
||||
head = git_text(path, "rev-parse", "--verify", "HEAD")
|
||||
live = run(
|
||||
("git", "ls-remote", "--exit-code", "--heads", "origin", "refs/heads/main"),
|
||||
cwd=path,
|
||||
)
|
||||
lines = live.stdout.strip().splitlines()
|
||||
if live.returncode or len(lines) != 1:
|
||||
raise MetaPreparationError("Could not verify live preparation source main.")
|
||||
remote_main, separator, reference = lines[0].partition("\t")
|
||||
if (
|
||||
not _OBJECT.fullmatch(head)
|
||||
or not _OBJECT.fullmatch(remote_main)
|
||||
or not separator
|
||||
or reference != "refs/heads/main"
|
||||
or git(path, "merge-base", "--is-ancestor", remote_main, head).returncode
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Preparation source main diverged; fetch and review separately."
|
||||
)
|
||||
sources[name] = {
|
||||
"head": head,
|
||||
"remote_main": remote_main,
|
||||
"filesystem": filesystems[name],
|
||||
}
|
||||
tag = f"v{target_version}"
|
||||
published = remote_tag_commit(repo_path, remote="origin", tag=tag)
|
||||
if (
|
||||
published.error
|
||||
or published.tag_object
|
||||
or git_text(repo_path, "rev-parse", "--verify", f"refs/tags/{tag}")
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"An existing or unverifiable target Meta tag blocks source preparation."
|
||||
)
|
||||
inputs, payloads = {}, {}
|
||||
total = 0
|
||||
for path in sorted(paths):
|
||||
payload, identity = _read_input(path)
|
||||
total += len(payload)
|
||||
if total > MAX_TOTAL_BYTES:
|
||||
raise MetaPreparationError(
|
||||
"Developer composition exceeds its aggregate input bound."
|
||||
)
|
||||
relative = path.relative_to(workspace).as_posix()
|
||||
inputs[relative], payloads[relative] = identity, payload
|
||||
generator_path = META_ROOT / "tools/release/generate-developer-meta-package.py"
|
||||
_trusted_ancestry(generator_path.parent)
|
||||
_owned_path(META_ROOT, directory=True)
|
||||
generator, generator_identity = _read_input(generator_path)
|
||||
if total + len(generator) > MAX_TOTAL_BYTES:
|
||||
raise MetaPreparationError(
|
||||
"Developer composition exceeds its aggregate input bound."
|
||||
)
|
||||
payloads[GENERATOR] = generator
|
||||
current = tomllib.loads(payloads[f"govoplan/{PACKAGE}"].decode("utf-8")).get(
|
||||
"project", {}
|
||||
)
|
||||
core = tomllib.loads(payloads["govoplan-core/pyproject.toml"].decode("utf-8")).get(
|
||||
"project", {}
|
||||
)
|
||||
if (
|
||||
not isinstance(current, dict)
|
||||
or current.get("name") != "govoplan"
|
||||
or not isinstance(current.get("version"), str)
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Nested developer-package identity and version must be exact."
|
||||
)
|
||||
if (
|
||||
not isinstance(core, dict)
|
||||
or core.get("name") != "govoplan-core"
|
||||
or core.get("version") != target_version
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Prepare and commit Core at the requested target version before Meta preparation."
|
||||
)
|
||||
from .version_alignment import repository_version_issues
|
||||
|
||||
if repository_version_issues(
|
||||
workspace / "govoplan-core", expected_version=target_version
|
||||
):
|
||||
raise MetaPreparationError(
|
||||
"Prepare aligned Core version metadata before Meta preparation."
|
||||
)
|
||||
receipt = {
|
||||
"kind": "developer_meta_preparation_v1",
|
||||
"workspace": str(workspace),
|
||||
"target_version": target_version,
|
||||
"sources": sources,
|
||||
"inputs": inputs,
|
||||
"operator_generator": generator_identity,
|
||||
}
|
||||
return receipt, payloads
|
||||
|
||||
|
||||
def _render(payloads: dict[str, bytes]) -> bytes:
|
||||
# The trusted operator generator sees only the frozen bounded data snapshot.
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-meta-render-") as temporary:
|
||||
workspace = Path(temporary)
|
||||
for relative, payload in payloads.items():
|
||||
path = workspace / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(payload)
|
||||
render = runpy.run_path(str(workspace / GENERATOR))["render"]
|
||||
result = render(
|
||||
workspace=workspace,
|
||||
requirements=workspace / "govoplan/requirements-release.txt",
|
||||
).encode("utf-8")
|
||||
if len(result) > MAX_INPUT_BYTES:
|
||||
raise MetaPreparationError(
|
||||
"Generated developer package exceeds its output bound."
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def preview_meta_mutation(*, repo_path: Path, target_version: str):
|
||||
version = normalize_version(target_version)
|
||||
if not version:
|
||||
raise MetaPreparationError("A valid target release version is required.")
|
||||
receipt, payloads = _snapshot(repo_path=repo_path, target_version=version)
|
||||
after = _render(payloads)
|
||||
observed, _ = _snapshot(repo_path=repo_path, target_version=version)
|
||||
if observed != receipt:
|
||||
raise MetaPreparationError(
|
||||
"Developer preparation inputs changed during preview."
|
||||
)
|
||||
receipt["output_sha256"] = hashlib.sha256(after).hexdigest()
|
||||
return receipt, payloads[f"govoplan/{PACKAGE}"], after
|
||||
|
||||
|
||||
def prepare_developer_meta_package(
|
||||
*,
|
||||
repo_path: Path,
|
||||
target_version: str,
|
||||
apply: bool = False,
|
||||
expected_receipt=None,
|
||||
confirm_out_of_run: bool = False,
|
||||
) -> dict:
|
||||
"""Preview or explicitly apply one full generated file; never commit/publish."""
|
||||
try:
|
||||
receipt, before, after = preview_meta_mutation(
|
||||
repo_path=repo_path, target_version=target_version
|
||||
)
|
||||
result = {
|
||||
"status": "planned" if before != after else "noop",
|
||||
"path": PACKAGE,
|
||||
"receipt": receipt,
|
||||
"after_sha256": hashlib.sha256(after).hexdigest(),
|
||||
"changed": before != after,
|
||||
}
|
||||
if not apply:
|
||||
return result
|
||||
if not confirm_out_of_run:
|
||||
raise MetaPreparationError(
|
||||
"Explicitly confirm that no durable run is active for this source workspace."
|
||||
)
|
||||
if receipt != expected_receipt:
|
||||
raise MetaPreparationError(
|
||||
"Preparation inputs changed since the reviewed preview; create a fresh preview."
|
||||
)
|
||||
if before == after:
|
||||
return result
|
||||
from .version_metadata import _atomic_write
|
||||
|
||||
source_receipt = {
|
||||
key: value for key, value in receipt.items() if key != "output_sha256"
|
||||
}
|
||||
observed, _ = _snapshot(
|
||||
repo_path=repo_path, target_version=receipt["target_version"]
|
||||
)
|
||||
if observed != source_receipt:
|
||||
raise MetaPreparationError(
|
||||
"Preparation inputs changed before the file effect; create a fresh preview."
|
||||
)
|
||||
try:
|
||||
_atomic_write(repo_path / PACKAGE, after)
|
||||
observed, payloads = _snapshot(
|
||||
repo_path=repo_path,
|
||||
target_version=receipt["target_version"],
|
||||
output_dirty=True,
|
||||
)
|
||||
comparison = copy.deepcopy(observed)
|
||||
output = f"govoplan/{PACKAGE}"
|
||||
comparison["inputs"][output] = receipt["inputs"][output]
|
||||
if comparison != source_receipt or payloads[output] != after:
|
||||
raise MetaPreparationError("Preparation inputs changed after writing.")
|
||||
except Exception as exc:
|
||||
raise MetaPreparationAmbiguous(
|
||||
"Generated file may have been written but its write/post-check failed; "
|
||||
"review the delta and reconcile manually."
|
||||
) from exc
|
||||
return {**result, "status": "prepared", "after_receipt": observed}
|
||||
except MetaPreparationError:
|
||||
raise
|
||||
except (OSError, UnicodeError, ValueError, KeyError, TypeError) as exc:
|
||||
raise MetaPreparationError(
|
||||
f"Developer preparation failed closed ({type(exc).__name__})."
|
||||
) from exc
|
||||
Reference in New Issue
Block a user