diff --git a/docs/operations/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md b/docs/operations/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md index dee3a87..3155781 100644 --- a/docs/operations/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md +++ b/docs/operations/INSTALLATION_AND_DEPLOYMENT_ARCHITECTURE.md @@ -361,6 +361,15 @@ and WebUI API proxy traffic across API replicas. The WebUI and API services do not publish host ports. HAProxy has no Docker socket and discovers only the bounded replica slots rendered into `load-balancer.cfg`. +New installation specifications default to HAProxy `3.2.23-alpine`, pinned to +registry index `sha256:6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e`. +This patch remains on HAProxy 3.2 LTS and Alpine 3.24.1. Loading an existing +specification preserves its explicit image; it does not perform an upgrade. +The [runtime remediation evidence](../security/RUNTIME_IMAGE_REMEDIATION_2026-09-08.md) +records both architecture scans and the pending binary/configuration, runtime, +inventory and final-image checks. The source default is not a release approval: +runtime publication remains held in Meta #52. + Replica counts are desired state: ```sh diff --git a/docs/operations/PACKAGE_REGISTRY_RELEASES.md b/docs/operations/PACKAGE_REGISTRY_RELEASES.md index 6d4597d..f79b118 100644 --- a/docs/operations/PACKAGE_REGISTRY_RELEASES.md +++ b/docs/operations/PACKAGE_REGISTRY_RELEASES.md @@ -263,11 +263,198 @@ python tools/release/generate-developer-meta-package.py python tools/release/generate-developer-meta-package.py --check ``` +The direct generator is a development synchronization tool, not a receipt-gated +release executor. For release preparation, use the guarded out-of-run stage below. + `push-release-tag.sh` performs this synchronization before release commits and tags. The meta-package is for editable/developer setup and composition tests. It does not enable modules, apply migrations, provision services, or establish backup and recovery evidence. +### Shared source-tag contract and Meta composition + +The shared version collector names Meta's real +`packages/govoplan-meta/pyproject.toml` separately from root `pyproject.toml`. +Only the registered `govoplan` system/meta repository with nested project name +`govoplan` receives this contract. Missing, unknown, or misidentified metadata +does not become a versionless exception. Version alignment compares the complete +nested file with the canonical operator-tool generator output: its version must +match Core, and dependencies and `full` composition must match the reviewed +requirements and workspace package versions. Validation never executes a +generator from a selected checkout. The shared trusted manifest checker can +load reviewed application manifests; these checks are not a code sandbox. + +Meta's complete generated file is recognized by shared version-mutation discovery, +but the generic durable version executor deliberately cannot write it. A durable +run freezes the release console's own Meta checkout as trusted runtime code; +changing it in place would invalidate that run. The planner therefore places Meta +after Core and exposes only non-executable support preparation/publication steps, +not misleading automatic Meta version, commit, tag, or push actions. A missing or +different Core target produces an actionable preparation prerequisite. + +Prepare Core and the intended module inputs first, commit their reviewed state, +then stop active durable runs for the target workspace. Use trusted operator tools +against a separate registered, private source checkout, never the running operator +Meta directory. Preview outside any selected source checkout, for example: + +```sh +python tools/release/prepare-developer-meta-package.py \ + --workspace /private/release-workspace --target-version X.Y.Z \ + > /private/operator/meta-preview.json +python tools/release/prepare-developer-meta-package.py \ + --workspace /private/release-workspace --target-version X.Y.Z \ + --receipt /private/operator/meta-preview.json --apply --confirm-out-of-run +``` + +The explicit confirmation attests that no durable run is active for that target +workspace; the helper does not discover or stop other processes. Preview/apply +requires registered clean main sources, matching origins and live-main ancestry, +Core already aligned at the target, the exact nested identity, and no existing or +unverifiable target Meta tag. Frozen receipts cover source HEADs/filesystem +identities, release requirements, every discovered registered full-composition +pyproject, the trusted generator snapshot, and the resulting full-file hash. +Inputs are limited to 128 selected files, 2 MiB per file and 16 MiB aggregate. +The canonical generator renders copied bounded data in a temporary directory; +no generator from the selected checkout executes. Changed receipts block before +the file effect. Apply writes only `packages/govoplan-meta/pyproject.toml`, then +rechecks the other sources and exact output. A write or post-check failure that +may have changed the file reports `needs-reconciliation` and leaves that bounded +delta for explicit review; it never retries, rolls back, commits or publishes. + +Review the complete generated composition and manually commit the resulting file. +Complete matching Core publication before guarded Meta source tagging/publication, +then start a fresh durable run from reviewed, clean, published operator tooling. +Hot self-updating durable Meta release execution remains explicitly unsupported; +this out-of-run preparation is the existing developer-meta support contract. + +For every `tag_repositories` batch, strict checks apply to every selected +repository before any tag creation, fetch, or push: registered checkout and +origin/push URL, a clean `main` tracking `origin/main`, live remote-main ancestry, +exact frozen HEADs, and immutable annotated local/remote tag objects. Git metadata +must remain inside the operator's trusted workspace. Missing local knowledge of +live remote main is a blocker; fetch and review it separately. Unknown repositories +and non-registered remote aliases fail closed. If selected, Meta runs last. +For Meta only, the matching annotated Core release must exist before its effect; Core may be +an earlier selected repository, or an already tagged dependency. Publication +requires that Core's exact tag and main commit are already remote. + +Non-Meta selections do not acquire Meta's composition or Core-tag prerequisite. +Local module-candidate tags still work before Core's final release lock or tag. +The existing Core WebUI bundle gate still applies to module publication and +batches selecting Core: relevant Core release-package and release-lock inputs +must be operator-owned regular files, at most 16 MiB each, and their identities +and content hashes are frozen before preflight and rechecked before every effect. +When Core is unselected, this does not require its checkout to be clean or tagged; +reviewed pending composition inputs retain their previous meaning. Backend-only +selections never read irrelevant Core WebUI files. + +Before even read-only Git commands, source ancestry must be owned by root or the +current operator and must not be group/world writable. A sticky shared ancestor +such as `/tmp` is permitted only above an owned, protected child; the workspace +and checkouts receive no writable-directory exception. The current operator must +own source inputs and actual Git/worktree/common metadata, which must be regular +files/directories, non-symlinked, and non-writable by other users. Metadata walks +are bounded to 500,000 entries and 128 levels, and tracked inputs to 100,000 paths +and 16 MiB of listing text. Read-only Git targets, object alternates/grafts and +hidden/sparse/unmerged index entries are blocked. Frozen receipts include actual +checkout/Git directory paths, devices, inodes, owners and modes, so replacing Git +metadata with the same HEAD is still detected. All selected version/composition +inputs must be tracked, including root and WebUI package/lock metadata, discovered +module manifests and package initializers, and Meta's nested package and release +requirements; ignored working files cannot supply declarations absent from a tag. +No chmod, ownership repair or +global Git trust change is performed. A shared writable workspace must first be +recreated or reviewed in the operator's protected release area by an explicitly +authorized preparation workflow. + +Preview is read-only. Local-tag mode creates only pinned annotated tags (or +retrieves an identical published annotation); it does not publish main or tags. +Publish mode atomically pushes the frozen main commit and annotation object, +without force, retagging, fallback, or automatic retry. The complete source +receipt is rechecked before every effect and afterward; remote main and the +exact annotated tag must both match, not merely the Git exit status. Changes +after preflight stop the remaining batch. Atomicity is per repository, not +across repositories: earlier successful publications and a newly created local +tag can remain after a later failure. Inspect reported receipts and obtain a new +review before retrying; do not move immutable tags. + +Whole-batch revalidation deliberately repeats source and live-remote checks around +each repository effect; the number of checks can grow quadratically with batch +size. Plan release time accordingly rather than bypassing trust checks. The +shared internal preflight is read-only and exposes no legacy mutation path. +The fixture suite covers Meta and non-Meta preview/local-tag/ +publication using temporary local bare remotes, including stale compositions, +unsafe origins, divergent branches, damaged tag identity, changed receipts and +false publication success. This is local tooling evidence, not a real release +publication or production permission check. + +Deutsch: Die gemeinsamen Helfer erkennen ausschließlich das registrierte +Meta-Repository mit dem echten Paket `packages/govoplan-meta/pyproject.toml` +(Projektname `govoplan`). Version und vollständige Zusammensetzung müssen dem +kanonischen Generator, Core und den geprüften Anforderungen entsprechen; der +Generator stammt niemals aus dem ausgewählten Checkout. Der gemeinsame +Manifestprüfer kann geprüften Anwendungscode laden und ist keine Sandbox. +Die gemeinsame Änderungsplanung erkennt die vollständig generierte Paketdatei, +aber der dauerhafte Versionsausführer darf Meta nicht selbst verändern: sein +eingefrorener Lauf bindet den Meta-Checkout als vertrauenswürdigen Programmstand. +Meta erscheint deshalb nach Core ausschließlich mit nicht automatisch ausführbaren +Vorbereitungs-/Veröffentlichungsschritten. Zuerst Core und Modulquellen vorbereiten +und geprüft committen; bei abweichender Core-Zielversion nennt der Plan diese +Voraussetzung ausdrücklich. Aktive dauerhafte Läufe des Ziel-Workspaces beenden. +Mit `prepare-developer-meta-package.py` zunächst eine Vorschau außerhalb der +Quell-Checkouts speichern, dann deren JSON über `--receipt` zusammen mit `--apply` +und `--confirm-out-of-run` bestätigen. Das Ziel muss ein separater registrierter +privater Checkout sein, niemals das laufende Operator-Meta. Die Bestätigung ist +eine Betreibererklärung; der Helfer sucht oder beendet keine fremden Prozesse. +Quell-HEADs, Pfadidentitäten, Anforderungen, alle registrierten vollständigen +Paket-Eingaben, der vertrauenswürdige Generator und der vollständige Ausgabehash +werden eingefroren. Es gelten höchstens 128 Quelldateien, 2 MiB je Datei und +16 MiB insgesamt. Core muss bereits vollständig zur Zielversion passen; vorhandene +oder nicht verifizierbare Meta-Zieltags sperren die Vorbereitung. Geänderte +Nachweise stoppen vor dem Schreiben. Ausschließlich die verschachtelte Paketdatei +wird vollständig generiert und danach geprüft; ein Fehler nach dem Schreiben +meldet `needs-reconciliation` und erfordert die manuelle Prüfung dieser begrenzten +Änderung, ohne automatisches Zurücksetzen. Kein automatischer +Commit, Push oder Wiederholungsversuch findet statt. Zusammensetzung prüfen, +manuell committen, Core zuerst veröffentlichen, dann die geschützte Meta-Tag-Route +verwenden und einen neuen dauerhaften Lauf starten. Eine Selbstaktualisierung +des aktiven dauerhaften Meta-Laufs bleibt ausdrücklich nicht unterstützt. +Für jeden Tag-Stapel, auch ohne Meta, gelten +Vertrauens-, Origin-, saubere Main- und Live-Abstammungsprüfungen für die gesamte +Auswahl vor jeder Änderung. Unbekannte Repositories und nicht registrierte +Remote-Aliase sind gesperrt. Nur bei ausgewähltem Meta gelten zusätzlich dessen +Zusammensetzungsprüfung und der passende annotierte Core-Tag als Voraussetzung; +Meta folgt zuletzt. Für Metas Veröffentlichung müssen Core-Tag und Main-Commit +bereits auf dem Remote vorliegen. Lokale Modul-Kandidatentags bleiben vor Cores +abschließendem Release-Lock und Tag möglich. Die vorhandene Core-WebUI-Bundleprüfung +bleibt bei Modulveröffentlichung und Core-Auswahl erhalten. Relevante Core-Paket- +und Lockdateien müssen eigene reguläre Dateien mit höchstens je 16 MiB sein; +Identität und Inhaltshash werden eingefroren und vor jeder Aktion erneut geprüft. +Nicht ausgewähltes Core benötigt dafür weder einen sauberen Checkout noch einen +Tag. Reine Backend-Auswahlen lesen keine irrelevanten Core-WebUI-Dateien. +Vor Git-Aufrufen werden Eigentümer, Schreibrechte, sichere +Pfadabstammung und echte Git-/Worktree-Metadaten geprüft; veränderbare gemeinsame +Verzeichnisse, fremde Eigentümer, Alternates, Grafts und versteckte Indexeinträge +sind gesperrt. Ein Sticky-Bit-Vorfahr wie `/tmp` ist nur oberhalb eines eigenen +geschützten Unterverzeichnisses zulässig. Es erfolgen weder Rechtereparaturen +noch globale Git-Vertrauensänderungen. Ausgewählte Versions- und Zusammensetzungs- +dateien müssen versioniert sein: Paket-/Lockdateien, Modulmanifeste und +Paketinitialisierer sowie Metas verschachteltes Paket und Release-Anforderungen. +Ignorierte Arbeitsdateien dürfen keine vom Tag abweichenden Angaben liefern. +Die Vorschau schreibt nichts, lokale Tags veröffentlichen nichts, +und die Veröffentlichung überträgt Main und den exakten annotierten Tag atomar +je Repository. Unmittelbar vor und nach den Aktionen werden die eingefrorenen +Quellnachweise erneut geprüft, einschließlich entferntem Main und Tag-Objekt. +Bei Änderungen oder Fehlern stoppt der Rest des Stapels ohne automatischen +Wiederholungsversuch. Frühere Veröffentlichungen und neu erzeugte lokale Tags +können bestehen bleiben: vor einem neuen Versuch Nachweise prüfen und erneut +freigeben, niemals unveränderliche Tags verschieben. Die vollständigen Quell- und +Live-Remote-Prüfungen werden um jede Aktion wiederholt; bei großen Stapeln kann +deren Anzahl quadratisch wachsen. Diese konservativen Prüfkosten gehören zur +Release-Planung. Der interne Vorprüfer ist ausschließlich lesend und besitzt +keinen alten Änderungspfad. Tests für Auswahlen mit und ohne Meta verwenden +nur temporäre lokale Remotes und ersetzen keine echte Veröffentlichungsprüfung. + If the tag-triggered developer meta-package job fails before publication, rerun `publish-developer-meta-package.yml` with the existing protected version. The manual path validates that tag against `main`, checks out its exact commit, and diff --git a/docs/security/RUNTIME_IMAGE_REMEDIATION_2026-09-08.md b/docs/security/RUNTIME_IMAGE_REMEDIATION_2026-09-08.md new file mode 100644 index 0000000..c14eb58 --- /dev/null +++ b/docs/security/RUNTIME_IMAGE_REMEDIATION_2026-09-08.md @@ -0,0 +1,139 @@ +# Runtime image remediation follow-up — 8 September 2026 + +Canonical tracking: [Meta #52](https://git.add-ideas.de/GovOPlaN/govoplan/issues/52) +and [website #9](https://git.add-ideas.de/add-ideas/addideas-govoplan-website/issues/9). +This addendum supplements the [original audit](RUNTIME_IMAGE_AUDIT_2026-09-08.md); +it does not replace that historical baseline or lift either publication or +deployment gate. The immutable 0.1.45 release and `catalog-v0.1.45` are unchanged. + +## Source change and candidate decisions + +New installer specifications now use +`haproxy:3.2.23-alpine@sha256:6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e`. +This is a patch update from 3.2.21 within the supported +[3.2 LTS branch](https://www.haproxy.org/), keeping Alpine 3.24.1. +The [publisher's exact build source](https://github.com/docker-library/haproxy/blob/7a5c202cde713867a737033dca56e7a211a8b8df/3.2/alpine/Dockerfile) +and scanned image configuration retain the non-root `haproxy` user, +`/usr/local/etc/haproxy/haproxy.cfg`, entrypoint and graceful-stop signal. +The [upstream changelog](https://www.haproxy.org/download/3.2/src/CHANGELOG) +includes HTTP parsing, TLS and memory-safety fixes in 3.2.22/3.2.23. +Existing specifications retain their explicit image, including an older pin; +this source change does not update a running installation. + +| Candidate | OS | C / H / M / L / Unknown, per architecture | Disposition | +| --- | --- | --- | --- | +| HAProxy `3.2.23-alpine` | Alpine 3.24.1 | 0 / 0 / 0 / 0 / 0 | Installer source default updated; binary/runtime checks pending | +| nginx-unprivileged `1.30.4-alpine` | Alpine 3.24.1 | 0 / 0 / 0 / 0 / 0 | Candidate only; compatibility and website OS upgrade review pending | +| Caddy `2.11.4-alpine` | Alpine 3.23.5 | 1 / 38 / 41 / 12 / 23 | Not selected; all 39 C/H records have recorded fixes | +| Node `24-alpine` (24.20.0) | Alpine 3.24.1 | 0 / 6 / 11 / 12 / 0 | Not selected; major change and six fixable HIGH records | + +Each row was scanned separately for **linux/amd64 and linux/arm64**, with the +same counts on both. Counts are package-vulnerability records, not distinct +CVEs or proven exploits. The [machine-readable evidence](runtime-image-candidates-2026-09-08.json) +contains exact index, platform-manifest, config and report digests, inventory +counts, scanner bounds and decisions. It is audit data, not an accepted release +manifest or an installer input. + +nginx's candidate reference is +`docker.io/nginxinc/nginx-unprivileged:1.30.4-alpine@sha256:442753882674b49ae2c1de83ed67896131c0777f56df5005e356e62bc3f7e7ce`. +It inventories 70 Alpine packages including nginx/NJS, uses UID 101 and exposes +8080. The [upstream stable release](https://nginx.org/en/download.html) and +[security advisories](https://nginx.org/en/security_advisories.html) include the +1.30.4 fixes. The publisher retains its +[unprivileged port and temporary-path contract](https://github.com/nginx/docker-nginx-unprivileged). +For the website, this changes nginx 1.27.5 to 1.30.4, NJS 0.8.10 to 1.0.1 and +Alpine 3.21.3 to 3.24.1. These changes are explicit review items; the website +Dockerfile has not been changed. The GovOPlaN Web image still requires an +explicit verified `NGINX_IMAGE` build argument. + +Caddy's candidate reference is +`docker.io/library/caddy:2.11.4-alpine@sha256:5f5c8640aae01df9654968d946d8f1a56c497f1dd5c5cda4cf95ab7c14d58648`. +Although this is the current +[official image line](https://raw.githubusercontent.com/docker-library/official-images/master/library/caddy), +its inventory still includes Go 1.26.3, `x/crypto` 0.52.0, `x/net` 0.55.0, +`x/text` 0.37.0 and gRPC 1.81.0. Recorded fixes include Go 1.26.6, +`x/crypto` 0.55.0, `x/net` 0.56.0, `x/text` 0.39.0 and gRPC 1.83.1; Alpine +findings also remain in c-ares, curl/libcurl and OpenSSL. The CRITICAL +`CVE-2026-56854` concerns `x/crypto/ssh` source-address enforcement. A module +record alone does not establish that this binary exposes that SSH path; exact +binary symbol/reachability analysis is still required for a disposition. + +The [official Node image catalogue](https://raw.githubusercontent.com/docker-library/official-images/master/library/node) +still maps Node 22 Alpine to 22.23.2 and the previously scanned digest. Node 24's +candidate is +`docker.io/library/node:24-alpine@sha256:e67514e5d0f6c46656005e1b693b2ec9d52e80b641307de684d4a015ba7a4eaf`. +Its HIGH records remain in two OpenSSL packages and npm dependencies +`brace-expansion`, `ip-address` and `tar`; fixing the earlier critical tar +record alone is insufficient. The website builder stays on Node 22 pending +a reviewed build-tool remedy and a final builder scan. + +## Method, verification and retained evidence + +The existing Trivy 0.74.0 executable was rehashed against the previously verified +archive member: `d89bcc6510a267f11b773398cbf1be5520ce39f9e8b6633178c4487f05b7d791`. +The same schema-2 vulnerability database was used, updated +`2026-09-07T19:06:01.154199452Z`. No tool installation or database refresh occurred. +Index bytes matched both the registry digest header and Docker Hub publisher +metadata; both platform-manifest byte hashes matched the index. All eight +registry-only scans completed successfully with validated JSON, `--list-all-pkgs`, +`--scanners vuln`, an eight-minute/2GB image bound, an empty Docker configuration +and no inherited credentials. Exit zero means execution succeeded. Private +temporary paths and in-memory artifact cache isolated this follow-up from the +earlier scanner's artifact cache; its vulnerability database was read only. + +Raw reports, logs, manifests, publisher metadata and the scanner script are in +`/home/zemion/.cache/govoplan-runtime-remediation.qfDSWHJh/`: + +- `summary.json` SHA-256: `0d44390408ab35270e4430516f77bf11aa7877334eff2ef19e11e9a863fe5c56`. +- `frozen-images.json` SHA-256: `d0cb156f4a88998531ec55ab950067a3f1350ded648f07650c463af101dad467`. +- `scan_successors.py` SHA-256: `f64c69e06efc2ad7b5a657a25aa73f9ff586e3685737d525456bcecbe3ab5f07`. + +Local retention is not permanent artifact hosting; preserve this evidence with +the eventual reviewed release. The JSON evidence records compressed registry +layer sizes; these are not expanded filesystem limits or final GovOPlaN sizes. + +Installer regression checks cover the new generated image pin, legacy +specification fallback, preserved explicit images, generated topology and +configuration: `python -I -m unittest discover -s tests -p +test_deployment_installer.py` ran 45 tests successfully with one skip because +Core was not importable in that isolated test environment. The skipped Core +startup-configuration integration was subsequently rerun in the shared development +environment with Core available: all 45 installer tests passed with no skips, +including generated-environment startup validation. This is configuration +validation, not execution of the candidate image. +Both repositories passed `git diff --check`; the audit JSON and all eight +report hashes were checked against the retained evidence. +**Docker, Podman and HAProxy executables are unavailable on +this host**, so no image or HAProxy configuration was executed and no daemon was +installed. Publisher metadata and installer tests support the scoped source +patch; they do not establish binary or deployed compatibility. + +## Gates that remain open + +- Validate `haproxy -c` on generated local, existing-proxy and managed-ingress + configurations using the exact pinned image and target architectures. Run + bounded isolated checks without live mounts, secrets, privilege or external + network access. Then verify DNS discovery, readiness, forwarded headers, + replica routing and graceful termination in the intended runtime. +- Test the nginx candidate with both the website configuration and GovOPlaN + WebUI entrypoint/proxy configuration, including UID 101, writable temporary + paths, health paths, cache headers and static catalog bytes. Approve the + website nginx/NJS/Alpine version changes before changing its Dockerfile. +- Resolve Caddy, Node build-tool and all unchanged baseline dependencies with + updated publisher images or narrow reviewed applicability evidence. No + severity-wide exceptions or custom third-party rebuilds were introduced. +- Close the original source-built/static inventory gaps. HAProxy's 24-package + OS inventory still omits the source-built HAProxy executable. Node's npm + inventory still omits the Node executable/stdlib. Garage, CPython, Redis, + PostgreSQL and QEMU gaps are unchanged. Alpine 3.24 EOL metadata is still + missing from this scanner; zero findings is not complete coverage. +- Scan **final built** API/Web/website layers and the selected managed + dependencies on both architectures, then perform migration, worker, + readiness and ingress smoke checks. Record failure and unknown states. + Secrets, misconfiguration and image signature policy need separate checks. +- Obtain the website deployment host/operator and rebuild/restart authority, + preserving the exact immutable catalog/keyring/module-directory bytes and + verifying fresh public responses after an authorized rollout. + +No images were built, executed, published or deployed; no running service, +release tag, signed manifest, CI image input or live infrastructure was changed. diff --git a/docs/security/SECURITY_FOLLOWUP_2026-09-08.md b/docs/security/SECURITY_FOLLOWUP_2026-09-08.md new file mode 100644 index 0000000..c386114 --- /dev/null +++ b/docs/security/SECURITY_FOLLOWUP_2026-09-08.md @@ -0,0 +1,219 @@ +# Security and performance follow-up — 8 September 2026 + +This follows the [original review](SECURITY_PERFORMANCE_REVIEW_2026-09-08.md) +and its post-release issue reconciliation. It describes new source work after +the frozen 0.1.45 release; it does not change published tags, packages, signed +catalogs or deployed images. Gitea remains the canonical state log. + +## Implemented source slices + +- [Core #297](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/297): + a shared disposable-process runner enforces wall/CPU/address-space/input/output + limits, bounded stderr, process-group cleanup and non-queuing per-process + admission. A private binary codec bounds decoding before allocating a full + object graph and preserves explicitly supported data types without pickle. + Read the owning Core `docs/BOUNDED_PROCESS_CONTRACT.md` before adding callers. +- Connectors XLSX parsing, Templates rendering, Files ZIP/TAR inspection and + extraction, and Dataflow reference previews/development execution now use + that boundary. Existing authorization, sessions, provider credentials, + idempotency and persistence remain in the parent. No unprotected inline + fallback is used. Each module contributes static EN/DE user/admin limits and + operational consequences through its manifest. +- Files snapshots authorized sources inside shared admission, validates private + staged members, and acknowledges each persisted member before decoding the + next. Numeric progress remains available. The acknowledgement is event-driven, + not a fixed sleep per member. Reads allocate by validated actual file size, + not by the configured ceiling. Failures reap children, clear private staging + and retain the existing transaction/blob cleanup and explicit retry behavior. +- Dataflow's normal reference preview formerly bypassed the backend wrapper; + it now enters the worker too. Nested source configurations cannot collide + merely because subflows reuse node IDs. Combined reference-source data is + checked before creating further columnar copies, while individual providers + retain their own authorized-read bounds. Staging/production still require + DuckDB; this change does not replace that separate backend. +- [Access #22](https://git.add-ideas.de/GovOPlaN/govoplan-access/issues/22): + current-password change, session/CSRF rotation, cross-tenant session and human + API-key revocation, and optional administrator-assisted recovery. Recovery + codes are hashed, single-use, expire after 15 minutes, require a current local + System owner and explicit identity verification, and recheck current account, + membership, tenant and issuer authority at redemption. A password change also + invalidates outstanding codes issued by that account for other people. Audit + evidence and validation/error responses do not contain passwords or codes. + External-provider and service-account rules remain separate. +- The Access UI provides first-login/required change, self-service change, + policy-aware sign-in help, public code redemption and eligible owner issuance. + Core consumes an optional lazy auth-action capability rather than importing + Access internals. The required-action gate fails closed if its UI is missing. +- [Workflow Engine #3](https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/3): + full-history lists batch pinned revisions, while new summary and bounded + step/event endpoints preserve authorization and explicit pagination. Existing + full-history responses are not silently truncated. Exact inbox total semantics + are retained and their counting cost is documented. +- [Meta #55](https://git.add-ideas.de/GovOPlaN/govoplan/issues/55): + shared version/planning helpers recognize the existing nested developer + package, not invented root metadata. All tag batches enforce trusted private + source ownership, registered origins and clean main/upstream state. Meta + batches additionally require exact composition and matching Core evidence. + Whole-batch preflight, + frozen source receipts, annotated immutable tags, object-pinned atomic + publication and post-effect remote checks are covered with temporary local + repositories. Hidden Git index flags, unsafe ancestry, alternates and changed + Git-directory identities are rejected. Selected version/composition metadata + must be tracked, so ignored files cannot describe bytes absent from a tag. + Applicable unselected Core WebUI inputs have bounded, frozen read receipts; + backend-only releases do not read them. The existing local module-candidate + exception remains intact. The weaker legacy mutation path was removed. + Canonical whole-package preview and receipt-bound apply now cover Meta's + version preparation too. Core must already match the target. Preparation + requires a separate trusted checkout, explicit out-of-run confirmation and + unchanged source/tooling receipts; it cannot rewrite the running operator. + Plans place Meta after Core and explain the manual preparation/publication + steps instead of promising a durable self-update. Ambiguous partial writes + require reconciliation, without automatic rollback or retry. +- [Runtime-image follow-up](RUNTIME_IMAGE_REMEDIATION_2026-09-08.md): eight new + registry-only scans cover four exact candidates on amd64 and arm64. New + installer specifications select the patched same-line HAProxy digest; + existing specifications retain their explicit image. Other candidates and + unresolved inventory/deployment gates remain visible, not blanket-approved. + +## Verification record + +Targeted checks include actual child execution, catastrophic regex CPU, +aggregate memory exhaustion, TAR extension metadata, noisy output, malformed +transport/staging data, Unicode allocation limits, cancellation/callback +failures, descendant cleanup, rollback and explicit retries. Local mixed-owner +composition tests completed nine real children, rejected six overlapping +requests as busy, observed at most one unreaped child and recovered all slots. +This is local admission evidence, not a target deployment load certification. + +Workflow fixtures serialize 40 different pinned revisions with five SQL reads; +summary lists use one query for 40 ordinary rows. Exact inbox totals for +40/400/4,000 candidates used one query, with measured local costs approximately +0.011/0.057/0.492 seconds. These are fixture measurements, not production SLOs. + +The broader Core API smoke suite exposed three stale campaign assertions. +All three failures were reproduced against the unchanged private frozen 0.1.45 +sources. Updated fixtures verify recipient-summary projection, detailed payload +separation and explicit fenced recovery of a confirmed stopped runtime; observing +SENDING alone must not make a claim recoverable. All 76 smoke tests then passed. +No production Campaign behavior was changed to satisfy these tests. + +The final release-tool suite passed 279 tests and 68 subtests, including +temporary local remotes and adversarial source/tag/receipt changes. Rechecking +the whole batch before effects is deliberately conservative: its repeated +filesystem/Git/remote work grows quadratically with batch size. It is not a +new unattended publication path or permission to execute unreviewed source. + +Strict interface inventory now reports no unclassified endpoints and exact +contextual help for all 133 high-risk controls. Seventeen password browser cases +include actual F1 help from the restricted screen, empty workspace scopes, +EN/DE layouts, Unicode boundaries and no credential values in help URLs. +The initial production bundle remains within the unchanged limits (512,036 +raw bytes and 162,415 gzip bytes; 1,713 gzip bytes below its ceiling), with +46 optional descriptors and no eager optional-module imports. + +The focused checker now includes the new Core process, mixed-owner admission, +Access password, Templates and Files worker tests, the repaired campaign smoke +cases, and browser-side auth/password transport contracts. The full focused run +passed, including 63 production module/build permutations and all 230 browser +cases. Its two opt-in Datasources PostgreSQL cases were skipped in that run +and subsequently passed against the isolated real database described below. +The final Meta preparation gate was added after that full run and verified +with the owning release-tool suite and the focused release-gate command. +Manifest validation passed for all 72 modules. The full focused log is +`/mnt/DATA/tmp/govoplan-security-followup-20260908-focused.log`. + +The first follow-up quick audit captured an unchanged 79-repository snapshot +in `/mnt/DATA/tmp/govoplan-security-followup-quick-20260908-7s8Sgh/`. +All four required scanners completed, with zero missing/execution reports; +all 168 report checksums and 163 machine-readable reports were validated. +Gitleaks found no secrets in all 79 histories and 79 worktrees. Local Semgrep +rules reported zero findings. Production Bandit reported 65 low and four medium +warnings, and production Ruff retained 54 warnings. The two added Bandit +warnings identify the new Core subprocess import and invocation: trusted +server-owned arguments, no shell, and the documented resource/process boundary +were reviewed; warnings remain visible. This is report-only evidence, not a +warning-free audit or a penetration test. A final snapshot follows the +cross-module declaration/contextual-help corrections and release-tool checks. + +That final audit completed on 8 September, 05:59:46–06:02:18 UTC, in +`/mnt/DATA/tmp/govoplan-security-final-quick-20260908-vAwQIh/`. All 79 start/end +source fingerprints were identical; all four scanners completed, all 168 +registered report checksums matched, and all 163 JSON/SARIF reports parsed. +There were no missing reports or scanner execution errors. Semgrep and both +Gitleaks scopes again reported zero findings. Production counts were unchanged +from the first follow-up: Bandit 65 low/four medium and Ruff 54. Test-only +counts were Bandit 140 low/34 medium and Ruff 136. A separate frozen scan of +all ten changed Meta release/deployment Python files reported seven low Bandit +and four Ruff S603 warnings, with no execution errors. Its four argv-only +subprocess sites were reviewed; the preparation additions introduced no new +warnings. No findings were hidden or severity-wide exceptions added. +The audit manifest SHA-256 is +`a997b786239cd11443cb665d5f9041a968cc38f9d49171e68bb868bf2bd73310`; +its report-checksum list SHA-256 is +`dc590ca5b0a4e445019a05536d410226088d67b40d61cd7657bdef4a4eae56d8`. + +The audit includes the eight committed feature/website source changes and the +final uncommitted Meta source. Only this evidence document was updated after +the source freeze ended; the final Meta commit and remote publication are +recorded in the linked Gitea issues, not inferred from local audit completion. + +Fresh dependency audits are retained in +`/mnt/DATA/tmp/govoplan-dependency-final-20260908-d24LEK/`: all four full npm +lockfile audits (Core WebUI, Mail root/WebUI and website) report zero known +vulnerabilities. Installed Python auditing covers 137 distributions with zero +known vulnerabilities; 51 local GovOPlaN distributions lack PyPI advisory +coverage. Core's 46 linked packages are likewise not claimed covered by public +registry advisories. All 12 dependency-file hashes and the installed inventory +were unchanged. No packages were installed or automatically fixed. + +Managed PostgreSQL 16.15 fixtures used private Unix sockets, synthetic roles +and databases, no TCP listener, per-case schemas and bounded SQL/lock waits. +Both previously skipped Datasources races passed. Twenty-one existing Access +password HTTP tests and four additional races passed on PostgreSQL: single-use +redemption, stale-session/password replacement, competing issuance, and issuer +password revocation during redemption. Four release/development migration checks +also passed for Access and Workflow, including credential preservation and +idempotent indexes. The four races are now owning opt-in Access regressions; +see `govoplan-access/docs/PASSWORD_RECOVERY_POSTGRES_TESTS.md`. These local +database checks do not certify a deployment, fleet load or external recovery +handover. With both explicit PostgreSQL test URLs enabled, the full Access suite +passed 122 tests and 18 subtests, and the full Datasources suite passed 57 tests, +without skips. Access retained 12 existing SQLite datetime-adapter warnings in +its separate SQLite migration cases. Both temporary PostgreSQL fixtures were +stopped and independently verified: no server process, private socket, +generated schema or synthetic cluster remains. Scripts, logs and shutdown +receipts are retained under `/home/zemion/.cache/govoplan-pg-security-20260908.RAUitg/` +and `/home/zemion/.cache/govoplan-pg-promoted-20260908.JZcIKL/`. + +## Adoption and remaining gates + +1. `AUTH_LOCAL_PASSWORD_RECOVERY_ENABLED` remains **false** by default. The + existing flag is still advisory until an operator explicitly adopts and + enables the complete recovery policy. Confirm who verifies identity and how + the one-time code is handed over; automated email recovery is not enabled. + Test first-login, lost-password, code expiry and administrator availability + in the target environment before enforcement. +2. Access migration `e9a2c5f8b1d4` adds recovery evidence; Workflow migration + `9e6b3f8a2c7d` adds summary-pagination indexes. Use normal backed-up upgrade + procedures and account for index-build cost. No manual live migration or + server restart was performed during this work. The user's existing devserver + has automatic reload, so live schema state must not be assumed unchanged. +3. Release preparation must assign new source/package versions and require a + Core version containing the new worker/auth contracts in the affected module + metadata, including matching WebUI assets. The old immutable release must + not be relabelled or treated as containing these APIs. +4. Resource limits are not an arbitrary-code, filesystem or network sandbox. + Admission is per API/worker process, not fleet-wide. Validate Linux/cgroup + memory, disk quotas, process counts, cancellation and legitimate large-file + workloads on the intended runtime before increasing concurrency. Core #297 + retains this target-evidence follow-up. +5. Meta #52 and website #9 retain runtime-image/publication/deployment holds. + Docker/Podman/HAProxy executables are unavailable here. Final built images, + binary/source inventories, ingress behavior, migration/readiness/worker + smoke checks and the website's target/operator authority remain outstanding. + Zero findings in a detected package inventory is not full image coverage. + +No real messages, IMAP appends, password resets, provider operations or deployment +actions were used as test fixtures. Development tests use temporary databases, +private temporary files, mock transports and managed test-browser servers. diff --git a/docs/security/SECURITY_PERFORMANCE_REVIEW_2026-09-08.md b/docs/security/SECURITY_PERFORMANCE_REVIEW_2026-09-08.md index bec5fc3..04d42e3 100644 --- a/docs/security/SECURITY_PERFORMANCE_REVIEW_2026-09-08.md +++ b/docs/security/SECURITY_PERFORMANCE_REVIEW_2026-09-08.md @@ -199,3 +199,9 @@ they do not change its immutable tags or published package bytes. These are unreleased follow-up source changes, not a new runtime release or deployment. The runtime-image hold under Meta #52 remains in force; the historical peer-dependency workaround still needs its separate review. + +Further implementation and adoption gates are tracked in the +[security follow-up](SECURITY_FOLLOWUP_2026-09-08.md), including disposable +parsing/execution workers, opt-in password recovery, workflow read projections +and the newer runtime-image evidence. The original scanner counts above remain +historical and are not silently replaced by later test results. diff --git a/docs/security/runtime-image-candidates-2026-09-08.json b/docs/security/runtime-image-candidates-2026-09-08.json new file mode 100644 index 0000000..b2fae55 --- /dev/null +++ b/docs/security/runtime-image-candidates-2026-09-08.json @@ -0,0 +1,305 @@ +{ + "schema_version": 1, + "purpose": "Audit evidence only; not a release manifest or active installer configuration.", + "observed_at": "2026-09-08T03:56:47.666808+00:00", + "runtime_publication_held": true, + "website_deployment_held": true, + "scan_execution_complete": true, + "coverage_complete": false, + "scanner": { + "name": "Trivy", + "version": "0.74.0", + "binary_sha256": "d89bcc6510a267f11b773398cbf1be5520ce39f9e8b6633178c4487f05b7d791", + "database_metadata": { + "Version": 2, + "NextUpdate": "2026-09-08T19:06:01.154199291Z", + "UpdatedAt": "2026-09-07T19:06:01.154199452Z", + "DownloadedAt": "2026-09-07T23:30:53.198039224Z" + }, + "source": "remote", + "scanners": [ + "vuln" + ], + "list_all_packages": true, + "images_executed": false, + "existing_docker_credentials_used": false, + "timeout": "8m", + "maximum_image_size": "2GB" + }, + "evidence": { + "private_directory": "/home/zemion/.cache/govoplan-runtime-remediation.qfDSWHJh", + "summary_sha256": "0d44390408ab35270e4430516f77bf11aa7877334eff2ef19e11e9a863fe5c56", + "frozen_images_sha256": "d0cb156f4a88998531ec55ab950067a3f1350ded648f07650c463af101dad467", + "scanner_script_sha256": "f64c69e06efc2ad7b5a657a25aa73f9ff586e3685737d525456bcecbe3ab5f07" + }, + "candidates": [ + { + "name": "haproxy", + "image": "docker.io/library/haproxy:3.2.23-alpine@sha256:6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e", + "disposition": "source_default_updated_binary_runtime_verification_pending", + "platforms": [ + { + "platform": "linux/amd64", + "manifest_digest": "sha256:0666a2c2f41d341084ed2da85392b48cdcd766adfa28231f31305724ed5c6ea5", + "config_digest": "sha256:9621d75e50a8f26d3738ae3cdbf15e98c6aa5cd48e6baca0699492de502156f5", + "compressed_layer_bytes": 20516844, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 24 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 0, + "MEDIUM": 0, + "LOW": 0, + "UNKNOWN": 0 + }, + "fixable_high_critical": 0, + "unique_cves": 0, + "report_sha256": "40cfc3db74e29f09723f38698660ccdd50ac935c8d6e1e75c6e0555b3e9361fb", + "log_sha256": "46881f695780f89c037d5b0b4dc0ded9ea4e459077c7658d80b786efc5754084" + }, + { + "platform": "linux/arm64", + "manifest_digest": "sha256:cd20b9dc6b4713956a2a043997001a1948167d345e7c8d5bd5ff2e667166651f", + "config_digest": "sha256:19796bff8905d4a46c9463c576203b20cac8984d41b7b01ea9cbff55e8422c34", + "compressed_layer_bytes": 20970772, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 24 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 0, + "MEDIUM": 0, + "LOW": 0, + "UNKNOWN": 0 + }, + "fixable_high_critical": 0, + "unique_cves": 0, + "report_sha256": "0e1326c58abf358d592fa55c94333228ce1094edf4e489fcc4ece6e32d3320f6", + "log_sha256": "397c2fbf1044cf50733d68b4b9cc4eb68211d96f1f302d5e9f8af0d11fb0de1c" + } + ] + }, + { + "name": "nginx-stable", + "image": "docker.io/nginxinc/nginx-unprivileged:1.30.4-alpine@sha256:442753882674b49ae2c1de83ed67896131c0777f56df5005e356e62bc3f7e7ce", + "disposition": "candidate_pending_compatibility", + "platforms": [ + { + "platform": "linux/amd64", + "manifest_digest": "sha256:b8c179cd3c2ae222a873dd59fbae240fadc03836cae5198afc9e9c19919c3880", + "config_digest": "sha256:8b5953dae38d27a76bca22373bb920fd6ce8d9d7da21578d2926e678002de8a0", + "compressed_layer_bytes": 25526590, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 70 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 0, + "MEDIUM": 0, + "LOW": 0, + "UNKNOWN": 0 + }, + "fixable_high_critical": 0, + "unique_cves": 0, + "report_sha256": "3ad164dae3cdf891b12e41451b8a8e65a5e1688824a7c01d848e1274363e6bf9", + "log_sha256": "0f99ff3d9b4396de48655bf8299df30c14ba0c579f480d37baa7d2f6a4c11f1d" + }, + { + "platform": "linux/arm64", + "manifest_digest": "sha256:b6742a0cbd749add25346658991c3da06a8e38796949df120fed78db8c512576", + "config_digest": "sha256:4d8b10f5d2ff99e7aa5f161930d8a4693a86a26f288ec8c0d4cd47f2ef5af179", + "compressed_layer_bytes": 25892370, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 70 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 0, + "MEDIUM": 0, + "LOW": 0, + "UNKNOWN": 0 + }, + "fixable_high_critical": 0, + "unique_cves": 0, + "report_sha256": "506fdeb97525ed8e4d9ae538c42bab7aa2217f662a7135f0f12d16d20410c7a6", + "log_sha256": "c41ef9ea091d00f18c7a097404672591bee85e7477ae17d8f5ca771d8e42b2bb" + } + ] + }, + { + "name": "caddy", + "image": "docker.io/library/caddy:2.11.4-alpine@sha256:5f5c8640aae01df9654968d946d8f1a56c497f1dd5c5cda4cf95ab7c14d58648", + "disposition": "not_selected_remaining_fixable_findings", + "platforms": [ + { + "platform": "linux/amd64", + "manifest_digest": "sha256:98eb57d882ccd5213d1688764db10c1ca2c58a1ca3a6717a3411ad798f7a423a", + "config_digest": "sha256:af555904a0961945f16bb323a501457b13a4f7e9bde969b145b97da80b38ecbe", + "compressed_layer_bytes": 23907283, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.23.5" + }, + "inventory": [ + { + "type": "alpine", + "packages": 32 + }, + { + "type": "gobinary", + "packages": 146 + } + ], + "counts": { + "CRITICAL": 1, + "HIGH": 38, + "MEDIUM": 41, + "LOW": 12, + "UNKNOWN": 23 + }, + "fixable_high_critical": 39, + "unique_cves": 68, + "report_sha256": "e483352a1d5b9b97950dcf92e4dfcf4600f5b09306af3d0640b10ca229a07779", + "log_sha256": "9cd599d6dd8c101b421fdeb57036cec1f69f815d765a8bf1f850ec60061036f4" + }, + { + "platform": "linux/arm64", + "manifest_digest": "sha256:1172d4213087d3fc30bafc7ff2c2896180eb0c41ff7f75f315568fb36cabdcba", + "config_digest": "sha256:6b08c1b9858ca9a7d99c1da13c3695081e0e604c6cf214ca26a7ce0e2c4fd9b4", + "compressed_layer_bytes": 22722712, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.23.5" + }, + "inventory": [ + { + "type": "alpine", + "packages": 32 + }, + { + "type": "gobinary", + "packages": 146 + } + ], + "counts": { + "CRITICAL": 1, + "HIGH": 38, + "MEDIUM": 41, + "LOW": 12, + "UNKNOWN": 23 + }, + "fixable_high_critical": 39, + "unique_cves": 68, + "report_sha256": "20e04d820e3b27d57575ea177e523e23109fd83344cdf57e184a4d2fad27e803", + "log_sha256": "a1d9c37aa7948c137db64040d95e5930c5859a8acd71ac5bc41ff168e2b270c5" + } + ] + }, + { + "name": "node-lts", + "image": "docker.io/library/node:24-alpine@sha256:e67514e5d0f6c46656005e1b693b2ec9d52e80b641307de684d4a015ba7a4eaf", + "disposition": "not_selected_remaining_fixable_findings", + "platforms": [ + { + "platform": "linux/amd64", + "manifest_digest": "sha256:4caaaf42195bcd6f6f3559a413b20cb8f8ad089e231ee874cf7701643966689f", + "config_digest": "sha256:ee289c69ed1ac50a5a042112ea97f132800e2dd53e832da27784f00e45b3289c", + "compressed_layer_bytes": 58486244, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 18 + }, + { + "type": "node-pkg", + "packages": 146 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 6, + "MEDIUM": 11, + "LOW": 12, + "UNKNOWN": 0 + }, + "fixable_high_critical": 6, + "unique_cves": 19, + "report_sha256": "c927d995dde700c92027f6328dc6c273f0f1445cd8154301480757cb962e02b2", + "log_sha256": "c7dc1900cbe39c9f91e228b2770bcbd394ec2914d2b3879478295fc7f8f77ab3" + }, + { + "platform": "linux/arm64", + "manifest_digest": "sha256:d3724e44ee368606d753e0027eb8d2a94fc1f275e5d9e4620178a12edb655f5f", + "config_digest": "sha256:722cc1507731edf58a4c0bc3e29553c44ce5774d0849242c1b237abc79926a0a", + "compressed_layer_bytes": 58935654, + "scan_exit_code": 0, + "os": { + "Family": "alpine", + "Name": "3.24.1" + }, + "inventory": [ + { + "type": "alpine", + "packages": 18 + }, + { + "type": "node-pkg", + "packages": 146 + } + ], + "counts": { + "CRITICAL": 0, + "HIGH": 6, + "MEDIUM": 11, + "LOW": 12, + "UNKNOWN": 0 + }, + "fixable_high_critical": 6, + "unique_cves": 19, + "report_sha256": "e5f7799761f23cefc36929381b4186eeb3afb46a2a559c789d12a7eea5dc30d0", + "log_sha256": "fd24671f0da4d3b20dc8bcc2718531e6f6e19155d49bed15958965e21dd10813" + } + ] + } + ] +} diff --git a/tests/test_deployment_installer.py b/tests/test_deployment_installer.py index 5ff66ec..c7866e3 100644 --- a/tests/test_deployment_installer.py +++ b/tests/test_deployment_installer.py @@ -780,6 +780,11 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertIn("redis", compose["services"]) self.assertIn("worker", compose["services"]) self.assertIn("load-balancer", compose["services"]) + self.assertEqual( + "haproxy:3.2.23-alpine@sha256:" + "6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e", + compose["services"]["load-balancer"]["image"], + ) self.assertNotIn("test-mail", compose["services"]) self.assertEqual( ["127.0.0.1:8080:8080"], @@ -1145,8 +1150,26 @@ class DeploymentInstallerTests(unittest.TestCase): self.assertEqual(1, parsed.replicas.web) self.assertEqual(1, parsed.replicas.worker) self.assertEqual("managed", parsed.components.load_balancer.mode) + self.assertEqual( + default_spec().components.load_balancer.image, + parsed.components.load_balancer.image, + ) self.assertEqual("local", parsed.ingress.mode) + def test_load_balancer_patch_does_not_rewrite_an_existing_image(self) -> None: + for image in ( + "haproxy:3.2.21-alpine", + "registry.example.test/haproxy@sha256:" + "a" * 64, + ): + with self.subTest(image=image): + saved = default_spec(load_balancer_image=image).to_dict() + + restored = parse_spec(json.loads(json.dumps(saved))) + compose = render_compose(restored) + + self.assertEqual(image, restored.components.load_balancer.image) + self.assertEqual(image, compose["services"]["load-balancer"]["image"]) + def test_compose_contains_no_secret_values(self) -> None: spec = default_spec() values = initial_secrets(spec) diff --git a/tests/test_isolated_work_composition.py b/tests/test_isolated_work_composition.py new file mode 100644 index 0000000..d01a88d --- /dev/null +++ b/tests/test_isolated_work_composition.py @@ -0,0 +1,240 @@ +"""Local mixed-owner admission/recovery evidence, not production capacity certification. + +All inputs are synthetic and held in memory. The spawn observer temporarily +holds the admitted parent's handshake so the other owners encounter the same +occupied slot deterministically. Children perform real XLSX, template and +Dataflow work; there is no mocked process execution, database, or live service. +The non-queuing gate promises retryable rejection, not scheduler fairness. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO +import json +import os +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +try: + from openpyxl import Workbook + from govoplan_connectors.backend.tabular_adapters import ( + parse_managed_tabular_content, + ) + from govoplan_core.core.templates import TemplateRenderRequest + from govoplan_core.security import bounded_process + from govoplan_core.security.bounded_process import ProcessBudgetError + from govoplan_core.settings import settings + from govoplan_dataflow.backend.backends import execute_typed_graph + from govoplan_dataflow.backend.schemas import ( + GraphEdge, + GraphNode, + GraphPosition, + PipelineGraph, + ) + from govoplan_templates.backend.rendering import _render_payload +except ImportError as exc: + raise unittest.SkipTest( + "Mixed-owner isolation requires the optional module test environment." + ) from exc + + +class IsolatedWorkCompositionTests(unittest.TestCase): + def setUp(self) -> None: + workbook = Workbook() + workbook.active.append(["name"]) + workbook.active.append(["Ada"]) + stream = BytesIO() + workbook.save(stream) + workbook.close() + self.workbook = stream.getvalue() + self.graph = PipelineGraph( + nodes=[ + GraphNode( + id="source", + type="source.inline", + label="Source", + position=GraphPosition(x=0, y=0), + config={"source_name": "records", "rows": [{"name": "Ada"}]}, + ), + GraphNode( + id="output", + type="output", + label="Output", + position=GraphPosition(x=100, y=0), + config={}, + ), + ], + edges=[GraphEdge(id="edge", source="source", target="output")], + ) + + def xlsx(self): + rows, sheet = parse_managed_tabular_content( + self.workbook, + filename="synthetic.xlsx", + content_type=None, + delimiter=",", + sheet_name=None, + ) + self.assertEqual(rows, ({"name": "Ada"},)) + self.assertEqual(sheet, "Sheet") + return "xlsx" + + def templates(self): + payload, content_type, _pages = _render_payload( + SimpleNamespace(name="Synthetic template"), + SimpleNamespace( + content_text="Hello {{item.name}}", + content_html=None, + template_type="letter", + layout={}, + output_profiles=[], + ), + request=TemplateRenderRequest( + template_id="synthetic", output_format="text" + ), + items=({"name": "Ada"},), + ) + self.assertEqual(payload, b"Hello Ada") + self.assertEqual(content_type, "text/plain; charset=utf-8") + return "templates" + + def dataflow(self): + result = execute_typed_graph(self.graph, backend="reference") + self.assertEqual(result.rows, [{"name": "Ada"}]) + return "dataflow" + + def test_one_shared_slot_rejects_other_owners_and_all_retries_recover(self): + owners = { + "xlsx": self.xlsx, + "templates": self.templates, + "dataflow": self.dataflow, + } + processes = [] + modules = [] + hold_next = False + entered = threading.Event() + release = threading.Event() + observer_lock = threading.Lock() + maximum_unreaped = 0 + observer_timeouts = 0 + original_popen = bounded_process.subprocess.Popen + + def observe_spawn(*args, **kwargs): + nonlocal hold_next, maximum_unreaped, observer_timeouts + process = original_popen(*args, **kwargs) + with observer_lock: + processes.append(process) + modules.append(args[0][5]) + maximum_unreaped = max( + maximum_unreaped, sum(item.returncode is None for item in processes) + ) + should_hold = hold_next + hold_next = False + if should_hold: + entered.set() + if not release.wait(8): + # Return control so the real runner's normal timeout and + # process-group cleanup still own this child on test error. + observer_timeouts += 1 + return process + + def rejected(operation): + try: + operation() + except Exception as exc: + cause = exc + while cause is not None and not isinstance(cause, ProcessBudgetError): + cause = cause.__cause__ + self.assertIsInstance(cause, ProcessBudgetError) + self.assertEqual(cause.code, "busy") + return "busy" + self.fail( + "A different module admitted work while the shared slot was occupied." + ) + + started = time.monotonic() + busy_count = 0 + try: + with ( + patch.object(settings, "isolated_process_concurrency", 1), + patch.object(bounded_process.subprocess, "Popen", observe_spawn), + ThreadPoolExecutor(max_workers=3) as executor, + ): + for owner, operation in owners.items(): + with self.subTest(admitted_owner=owner): + entered.clear() + release.clear() + hold_next = True + holder = executor.submit(operation) + try: + self.assertTrue( + entered.wait(5), + "The admitted operation never spawned its real child.", + ) + children_before = len(processes) + others = [ + work for label, work in owners.items() if label != owner + ] + denied = [ + executor.submit(rejected, work) for work in others + ] + self.assertEqual( + [future.result(timeout=5) for future in denied], + ["busy", "busy"], + ) + busy_count += len(denied) + self.assertEqual(len(processes), children_before) + finally: + release.set() + self.assertEqual(holder.result(timeout=15), owner) + self.assertEqual(bounded_process._active, 0) + # Every rejected owner is retried through its real API. + # Each must complete after the previous holder releases. + for other in others: + other() + self.assertEqual(bounded_process._active, 0) + finally: + release.set() + for process in processes: + self.assertIsNotNone(process.returncode, "Worker was not reaped.") + self.assertTrue( + all( + stream.closed + for stream in (process.stdin, process.stdout, process.stderr) + ) + ) + with self.assertRaises(ChildProcessError): + os.waitpid(process.pid, os.WNOHANG) + self.assertEqual(maximum_unreaped, 1) + self.assertEqual(observer_timeouts, 0) + self.assertEqual(len(processes), 9) + self.assertEqual(busy_count, 6) + self.assertEqual( + set(modules), + { + "govoplan_connectors.backend.tabular_adapters", + "govoplan_templates.backend.rendering", + "govoplan_dataflow.backend.backends.reference", + }, + ) + print( + json.dumps( + { + "local_composition": { + "successful_children": len(processes), + "busy_rejections": busy_count, + "maximum_unreaped_children": maximum_unreaped, + "all_children_reaped": True, + "seconds": round(time.monotonic() - started, 3), + } + } + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_platform_interface_inventory.py b/tests/test_platform_interface_inventory.py index 8a124a8..273f3bb 100644 --- a/tests/test_platform_interface_inventory.py +++ b/tests/test_platform_interface_inventory.py @@ -96,6 +96,18 @@ class PlatformInterfaceInventoryTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "tracking_issue"): inventory._load_endpoint_declarations(path) + def test_bounded_workflow_read_apis_have_explicit_headless_declarations(self) -> None: + declarations = inventory._load_endpoint_declarations(inventory.DEFAULT_ENDPOINT_DECLARATIONS) + for path in ( + "/workflow/instances/summaries", "/workflow/instances/{}/summary", + "/workflow/instances/{}/steps", "/workflow/instances/{}/events", + ): + with self.subTest(path=path): + entry = declarations[("govoplan-workflow-engine", "GET", path)] + self.assertEqual("intentionally_headless", entry["category"]) + self.assertIn("current-authorized", entry["rationale"]) + self.assertIn("workflow.instance-history", entry["rationale"]) + def test_inventory_reports_unclassified_and_stale_endpoint_declarations( self, ) -> None: diff --git a/tests/test_release_meta_preparation.py b/tests/test_release_meta_preparation.py new file mode 100644 index 0000000..ad7366a --- /dev/null +++ b/tests/test_release_meta_preparation.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +from dataclasses import replace +from contextlib import redirect_stdout +import io +import json +from pathlib import Path +import runpy +import shutil +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools/release")) + +from govoplan_release import meta_preparation # noqa: E402 +from govoplan_release.git_state import collect_repository_snapshot # noqa: E402 +from govoplan_release.meta_preparation import ( # noqa: E402 + MetaPreparationError, + prepare_developer_meta_package, +) +from govoplan_release.model import RepositorySpec # noqa: E402 +from govoplan_release.selective_planner import build_selective_release_plan # noqa: E402 +from govoplan_release.version_metadata import ( # noqa: E402 + VersionMetadataError, + apply_version_metadata_mutations, + version_metadata_mutations, +) +import test_release_meta_source_tag as meta_fixture # noqa: E402 +from test_release_plan_guidance import dashboard # noqa: E402 +from test_release_repository_tag import create_release_repo, git, git_text # noqa: E402 + + +class MetaPreparationTests(unittest.TestCase): + synchronize = meta_fixture.MetaSourceTagTests.synchronize + + def setUp(self): + meta_fixture.MetaSourceTagTests.setUp(self) + self.operator = self.root / "operator" + self.generator = ( + self.operator / "tools/release/generate-developer-meta-package.py" + ) + self.generator.parent.mkdir(parents=True) + shutil.copyfile( + ROOT / "tools/release/generate-developer-meta-package.py", self.generator + ) + self.enterContext(patch.object(meta_preparation, "META_ROOT", self.operator)) + + def prepare_core(self): + apply_version_metadata_mutations(self.core, target_version="0.1.11") + git(self.core, "add", ".") + git(self.core, "commit", "-m", "Prepared synthetic Core target") + + def preview(self, **kwargs): + return prepare_developer_meta_package( + repo_path=self.meta, target_version="0.1.11", **kwargs + ) + + def apply(self, preview): + return self.preview( + apply=True, expected_receipt=preview["receipt"], confirm_out_of_run=True + ) + + def test_full_canonical_preview_apply_and_shared_mutation_discovery(self): + extra, remote = create_release_repo( + root=self.root, + workspace=self.workspace, + name="govoplan-workflow-engine", + version="0.2.3", + ) + self.specs.append( + { + "name": extra.name, + "path": extra.name, + "category": "module", + "subtype": "", + "remote": str(remote), + } + ) + self.registry.write_text(json.dumps({"repositories": self.specs})) + self.prepare_core() + before = self.package.read_bytes() + preview = self.preview() + self.assertEqual("planned", preview["status"]) + self.assertEqual(before, self.package.read_bytes()) + mutations = version_metadata_mutations(self.meta, target_version="0.1.11") + self.assertEqual([meta_preparation.PACKAGE], [item.path for item in mutations]) + self.assertIn(b"govoplan-workflow-engine==0.2.3", mutations[0].after) + self.assertIn(b"govoplan-core==0.1.11", mutations[0].after) + result = self.apply(preview) + self.assertEqual("prepared", result["status"]) + self.assertEqual(mutations[0].after, self.package.read_bytes()) + self.assertEqual( + self.render(workspace=self.workspace, requirements=self.requirements), + self.package.read_text(), + ) + self.assertEqual( + f"M {meta_preparation.PACKAGE}", + git_text(self.meta, "status", "--porcelain"), + ) + self.assertFalse(git_text(self.meta, "tag", "--list")) + + def test_core_target_must_already_be_prepared(self): + before = self.package.read_bytes() + with self.assertRaisesRegex(MetaPreparationError, "Prepare and commit Core"): + self.preview() + self.assertEqual(before, self.package.read_bytes()) + + def test_changed_requirements_receipt_blocks_before_any_output(self): + self.prepare_core() + preview = self.preview() + before = self.package.read_bytes() + self.requirements.write_text( + self.requirements.read_text() + "# reviewed different inputs\n" + ) + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Changed synthetic requirements") + with self.assertRaisesRegex(MetaPreparationError, "changed since"): + self.apply(preview) + self.assertEqual(before, self.package.read_bytes()) + + def test_source_change_immediately_before_effect_is_rechecked(self): + self.prepare_core() + preview = self.preview() + before = self.package.read_bytes() + original = meta_preparation.preview_meta_mutation + + def changed(**kwargs): + result = original(**kwargs) + self.requirements.write_text( + self.requirements.read_text() + "# concurrent change\n" + ) + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Concurrent synthetic change") + return result + + with patch.object( + meta_preparation, "preview_meta_mutation", side_effect=changed + ): + with self.assertRaisesRegex(MetaPreparationError, "changed before"): + self.apply(preview) + self.assertEqual(before, self.package.read_bytes()) + + def test_core_full_package_and_operator_generator_are_receipt_bound(self): + self.prepare_core() + for path, repository in ( + (self.core / "pyproject.toml", self.core), + (self.access / "pyproject.toml", self.access), + (self.generator, None), + ): + with self.subTest(input=path.name, repo=str(repository)): + preview = self.preview() + before = self.package.read_bytes() + path.write_text(path.read_text() + "\n# changed frozen input\n") + if repository is not None: + git(repository, "add", ".") + git( + repository, + "commit", + "-m", + "Changed synthetic composition input", + ) + with self.assertRaisesRegex(MetaPreparationError, "changed since"): + self.apply(preview) + self.assertEqual(before, self.package.read_bytes()) + + def test_post_write_source_change_is_reported_without_retry_or_rollback(self): + from govoplan_release import version_metadata + + self.prepare_core() + preview = self.preview() + original = version_metadata._atomic_write + + def changed(path, payload): + original(path, payload) + self.requirements.write_text( + self.requirements.read_text() + "# concurrent after write\n" + ) + + with patch.object( + version_metadata, "_atomic_write", side_effect=changed + ) as writer: + with self.assertRaisesRegex( + meta_preparation.MetaPreparationAmbiguous, "write/post-check failed" + ): + self.apply(preview) + self.assertEqual(1, writer.call_count) + self.assertIn('version = "0.1.11"', self.package.read_text()) + self.assertIn("# concurrent after write", self.requirements.read_text()) + + def test_write_failure_after_replace_requires_reconciliation(self): + from govoplan_release import version_metadata + + self.prepare_core() + preview = self.preview() + original = version_metadata._atomic_write + + def partial(path, payload): + original(path, payload) + raise OSError("Synthetic directory fsync failure after replacement") + + with patch.object(version_metadata, "_atomic_write", side_effect=partial) as writer: + with self.assertRaisesRegex(meta_preparation.MetaPreparationAmbiguous, "may have been written"): + self.apply(preview) + self.assertEqual(1, writer.call_count) + self.assertIn('version = "0.1.11"', self.package.read_text()) + + def test_cli_requires_reviewed_receipt_and_explicit_out_of_run_confirmation(self): + self.prepare_core() + main = runpy.run_path( + str(ROOT / "tools/release/prepare-developer-meta-package.py") + )["main"] + arguments = [ + "prepare-developer-meta-package.py", + "--workspace", + str(self.workspace), + "--target-version", + "0.1.11", + ] + output = io.StringIO() + with patch.object(sys, "argv", arguments), redirect_stdout(output): + self.assertEqual(0, main()) + preview = self.root / "meta-preview.json" + preview.write_text(output.getvalue()) + with ( + patch.object(sys, "argv", [*arguments, "--apply"]), + redirect_stdout(io.StringIO()), + ): + self.assertEqual(1, main()) + with ( + patch.object( + sys, + "argv", + [ + *arguments, + "--apply", + "--receipt", + str(preview), + "--confirm-out-of-run", + ], + ), + redirect_stdout(io.StringIO()), + ): + self.assertEqual(0, main()) + + def test_unknown_full_input_and_unsafe_operator_tooling_fail_closed(self): + self.prepare_core() + unknown = self.workspace / "govoplan-unknown/pyproject.toml" + unknown.parent.mkdir() + unknown.write_text('[project]\nname="govoplan-unknown"\nversion="1.0.0"\n') + with self.assertRaisesRegex(MetaPreparationError, "unregistered"): + self.preview() + unknown.unlink() + self.generator.chmod(0o666) + with self.assertRaisesRegex(MetaPreparationError, "owned, bounded regular"): + self.preview() + + def test_wrong_nested_identity_and_existing_immutable_tag_fail_closed(self): + self.prepare_core() + original = self.package.read_text() + self.package.write_text( + original.replace('name = "govoplan"', 'name = "not-govoplan"') + ) + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Wrong synthetic package identity") + with self.assertRaisesRegex(MetaPreparationError, "identity"): + self.preview() + self.package.write_text(original) + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Restore synthetic package identity") + git(self.meta, "tag", "-a", "v0.1.11", "-m", "Immutable target") + with self.assertRaisesRegex(MetaPreparationError, "target Meta tag"): + self.preview() + + def test_no_generic_durable_self_mutation_or_running_tooling_target(self): + self.prepare_core() + preview = self.preview() + with self.assertRaisesRegex(VersionMetadataError, "outside durable runs"): + apply_version_metadata_mutations(self.meta, target_version="0.1.11") + with self.assertRaisesRegex(MetaPreparationError, "confirm"): + self.preview(apply=True, expected_receipt=preview["receipt"]) + with patch.object(meta_preparation, "META_ROOT", self.meta): + with self.assertRaisesRegex(MetaPreparationError, "running operator"): + self.preview() + + def test_next_version_plan_is_actionable_core_first_without_meta_executor(self): + snapshots = tuple( + collect_repository_snapshot( + RepositorySpec(**spec), + workspace_root=self.workspace, + target_tag="v0.1.11", + ) + for spec in self.specs[:2] + ) + source = replace( + dashboard(workspace=self.workspace, version=self.version), + repositories=snapshots, + ) + plan = build_selective_release_plan( + source, + selected_repos=("govoplan", "govoplan-core"), + target_version="0.1.11", + ) + self.assertEqual( + ["govoplan-core", "govoplan"], [unit.repo for unit in plan.units] + ) + findings = [ + finding for finding in plan.gate_findings if finding.repo == "govoplan" + ] + self.assertEqual( + ["developer_meta_core_preparation_required"], + [finding.code for finding in findings], + ) + self.assertIn("prepare-developer-meta-package.py", findings[0].remediation) + meta_steps = [step for step in plan.dry_run_steps if step.repo == "govoplan"] + self.assertEqual( + ["govoplan:prepare-support", "govoplan:publish-support"], + [step.id for step in meta_steps], + ) + self.assertTrue(all(step.status == "needs-executor" for step in meta_steps)) + self.prepare_core() + prepared = build_selective_release_plan( + source, selected_repos=("govoplan",), target_version="0.1.11" + ) + self.assertEqual("developer_meta_out_of_run", prepared.gate_findings[0].code) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_meta_source_tag.py b/tests/test_release_meta_source_tag.py new file mode 100644 index 0000000..05898bf --- /dev/null +++ b/tests/test_release_meta_source_tag.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import runpy +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools/release")) + +from govoplan_release import source_tag_batch as meta_source_tag, workspace # noqa: E402 +from govoplan_release.git_state import collect_versions # noqa: E402 +from govoplan_release.model import RepositorySnapshot, RepositorySpec, VersionSnapshot # noqa: E402 +from govoplan_release.repository_tag import tag_repositories # noqa: E402 +from govoplan_release.selective_planner import build_unit # noqa: E402 +from govoplan_release.version_alignment import repository_version_issues # noqa: E402 +from test_release_repository_tag import ( # noqa: E402 + add_scoped_workflow_manifest, + create_release_repo, + git, + git_text, + ref_exists, +) + + +class MetaSourceTagTests(unittest.TestCase): + def setUp(self): + self.temporary = self.enterContext( + tempfile.TemporaryDirectory(prefix="meta-release-tests-") + ) + self.root = Path(self.temporary) + self.workspace = self.root / "workspace" + self.workspace.mkdir() + self.version = "0.1.10" + self.core, self.core_remote = create_release_repo( + root=self.root, + workspace=self.workspace, + name="govoplan-core", + version=self.version, + ) + self.access, self.access_remote = create_release_repo( + root=self.root, + workspace=self.workspace, + name="govoplan-access", + version=self.version, + ) + add_scoped_workflow_manifest(self.access) + self.meta = self.workspace / "govoplan" + self.meta_remote = self.root / "govoplan.git" + git(self.root, "init", "--bare", str(self.meta_remote)) + git(self.workspace, "init", "-b", "main", str(self.meta)) + git(self.meta, "config", "user.name", "Meta Release Fixture") + git(self.meta, "config", "user.email", "release@example.invalid") + self.package = self.meta / "packages/govoplan-meta/pyproject.toml" + self.package.parent.mkdir(parents=True) + self.requirements = self.meta / "requirements-release.txt" + self.requirements.write_text( + "../govoplan-core\ngovoplan-access @ git+ssh://git@example.invalid/GovOPlaN/govoplan-access.git@v0.1.10\n" + ) + self.render = runpy.run_path( + str(ROOT / "tools/release/generate-developer-meta-package.py") + )["render"] + self.synchronize() + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Nested developer package") + git(self.meta, "remote", "add", "origin", str(self.meta_remote)) + git(self.meta, "push", "-u", "origin", "main") + self.specs = [ + { + "name": name, + "category": "system" if subtype else "module", + "subtype": subtype, + "path": name, + "remote": str(remote), + } + for name, subtype, remote in ( + ("govoplan", "meta", self.meta_remote), + ("govoplan-core", "kernel", self.core_remote), + ("govoplan-access", "", self.access_remote), + ) + ] + self.registry = self.root / "repositories.json" + self.registry.write_text(json.dumps({"repositories": self.specs})) + self.enterContext(patch.object(workspace, "REPOSITORIES_FILE", self.registry)) + + def synchronize(self): + self.package.write_text( + self.render(workspace=self.workspace, requirements=self.requirements) + ) + + def commit_meta(self): + git(self.meta, "add", ".") + git(self.meta, "commit", "-m", "Changed synthetic metadata") + + def tag( + self, + *, + repos=("govoplan", "govoplan-core"), + apply=False, + push=False, + **overrides, + ): + return tag_repositories( + repos=repos, + repo_versions={repo: self.version for repo in repos}, + workspace_root=self.workspace, + apply=apply, + push=push, + **overrides, + ) + + def assert_no_tags(self): + for repo in ( + self.meta, + self.meta_remote, + self.core, + self.core_remote, + self.access, + self.access_remote, + ): + self.assertFalse(ref_exists(repo, "refs/tags/v0.1.10"), str(repo)) + + def test_explicit_nested_version_collection_and_alignment_without_root_package( + self, + ): + versions = collect_versions(self.meta) + self.assertIsNone(versions.pyproject) + self.assertEqual(self.version, versions.developer_meta) + self.assertEqual(self.version, versions.primary) + self.assertFalse((self.meta / "pyproject.toml").exists()) + self.assertEqual( + (), repository_version_issues(self.meta, expected_version=self.version) + ) + mismatch = repository_version_issues(self.meta, expected_version="0.1.11") + self.assertTrue( + any( + issue.source == "packages/govoplan-meta/pyproject.toml" + for issue in mismatch + ) + ) + + def test_planner_and_console_display_the_explicit_nested_version(self): + snapshot = RepositorySnapshot( + spec=RepositorySpec(**self.specs[0]), + absolute_path=str(self.meta), + exists=True, + is_git=True, + has_head=True, + branch="main", + versions=VersionSnapshot(developer_meta=self.version), + ) + unit = build_unit(snapshot, target_version=None, contracts=None) + self.assertEqual(self.version, unit.current_version) + self.assertEqual(self.version, unit.target_version) + html = (ROOT / "tools/release/webui/index.html").read_text() + self.assertIn( + "if (versions.developer_meta) return versions.developer_meta;", html + ) + drift = RepositorySnapshot( + spec=snapshot.spec, + absolute_path=str(self.meta), + exists=True, + is_git=True, + has_head=True, + branch="main", + versions=VersionSnapshot(pyproject="0.1.9", developer_meta=self.version), + ) + self.assertTrue( + any( + "version metadata is not aligned" in item + for item in build_unit( + drift, target_version=self.version, contracts=None + ).blockers + ) + ) + + def test_unknown_nested_package_and_missing_or_wrong_meta_identity_fail_closed( + self, + ): + unknown = self.workspace / "unknown" + nested = unknown / "packages/govoplan-meta/pyproject.toml" + nested.parent.mkdir(parents=True) + nested.write_text(self.package.read_text()) + self.assertIsNone(collect_versions(unknown).primary) + self.assertIn( + "no version metadata", + repository_version_issues(unknown, expected_version=self.version)[ + 0 + ].message, + ) + for value in ("", '[project]\nname="not-govoplan"\nversion="0.1.10"\n'): + with self.subTest(value=value): + self.package.write_text(value) + self.assertTrue( + repository_version_issues(self.meta, expected_version=self.version) + ) + + def test_preview_local_tag_and_publish_share_complete_nested_contract(self): + preview = self.tag(push=True) + self.assertEqual("planned", preview["status"], preview) + self.assertEqual( + ["govoplan-core", "govoplan"], + [row["repo"] for row in preview["repositories"]], + ) + self.assertEqual("registered-meta-batch-v1", preview["source_contract"]) + self.assert_no_tags() + local = self.tag(apply=True) + self.assertEqual("tagged", local["status"], local) + for repo in (self.core, self.meta): + self.assertEqual( + "tag", git_text(repo, "cat-file", "-t", "refs/tags/v0.1.10") + ) + self.assertFalse(ref_exists(self.meta_remote, "refs/tags/v0.1.10")) + published = self.tag(apply=True, push=True) + self.assertEqual("published", published["status"], published) + for repo, remote in ( + (self.core, self.core_remote), + (self.meta, self.meta_remote), + ): + self.assertEqual( + git_text(repo, "rev-parse", "HEAD"), + git_text(remote, "rev-parse", "refs/heads/main"), + ) + self.assertEqual( + git_text(repo, "rev-parse", "refs/tags/v0.1.10"), + git_text(remote, "rev-parse", "refs/tags/v0.1.10"), + ) + again = self.tag(apply=True, push=True) + self.assertEqual("published", again["status"], again) + + def test_stale_composition_blocks_whole_batch_before_local_tag_or_push(self): + self.package.write_text( + self.package.read_text().replace( + "govoplan-access==0.1.10", "govoplan-access==0.1.9" + ) + ) + self.commit_meta() + for apply, push in ((False, False), (True, False), (True, True)): + result = self.tag( + repos=("govoplan-core", "govoplan-access", "govoplan"), + apply=apply, + push=push, + ) + self.assertEqual("blocked", result["status"], result) + self.assert_no_tags() + + def test_core_outside_batch_requires_matching_existing_and_published_tag(self): + self.assertEqual("blocked", self.tag(repos=("govoplan",))["status"]) + git(self.core, "tag", "-a", "v0.1.10", "-m", "Core release") + self.assertEqual("planned", self.tag(repos=("govoplan",))["status"]) + self.assertEqual("blocked", self.tag(repos=("govoplan",), push=True)["status"]) + git(self.core, "push", "origin", "refs/tags/v0.1.10") + self.assertEqual("planned", self.tag(repos=("govoplan",), push=True)["status"]) + + def test_changed_core_version_and_explicit_selected_version_mismatch_block(self): + mismatch = tag_repositories( + repos=("govoplan", "govoplan-core"), + repo_versions={"govoplan": self.version, "govoplan-core": "0.1.11"}, + workspace_root=self.workspace, + apply=True, + push=True, + ) + self.assertEqual("blocked", mismatch["status"], mismatch) + (self.core / "pyproject.toml").write_text( + '[project]\nname="govoplan-core"\nversion="0.1.11"\n' + ) + git(self.core, "add", ".") + git(self.core, "commit", "-m", "Core new version") + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assert_no_tags() + + def test_unsafe_origin_on_any_selected_repo_blocks_every_effect(self): + for repo in (self.meta, self.core, self.access): + with self.subTest(repo=repo.name): + git( + repo, + "config", + "remote.origin.pushurl", + str(self.root / "unregistered.git"), + ) + result = self.tag( + repos=("govoplan-core", "govoplan-access", "govoplan"), + apply=True, + push=True, + ) + self.assertEqual("blocked", result["status"], result) + self.assertIn("registered origin", result["repositories"][0]["detail"]) + self.assert_no_tags() + git(repo, "config", "--unset", "remote.origin.pushurl") + + def test_world_writable_nonsticky_parent_blocks_without_changing_permissions(self): + original = self.root.stat().st_mode & 0o7777 + self.root.chmod(0o777) + try: + for apply in (False, True): + result = self.tag(apply=apply, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn( + "group/world writable", result["repositories"][0]["detail"] + ) + self.assertEqual(0o777, self.root.stat().st_mode & 0o7777) + self.assert_no_tags() + finally: + self.root.chmod(original) + + def test_wrong_metadata_owner_blocks_before_remote_lookup(self): + config = self.meta / ".git/config" + original = Path.lstat + + def wrong_owner(path, *args, **kwargs): + observed = original(path, *args, **kwargs) + if path == config: + fields = list(observed) + fields[4] = os.geteuid() + 1 + return os.stat_result(fields) + return observed + + with ( + patch.object(Path, "lstat", new=wrong_owner), + patch.object( + meta_source_tag, + "registered_source_origin_issues", + side_effect=AssertionError("must validate ownership before Git"), + ), + ): + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("current operator", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_hidden_index_flags_cannot_disguise_modified_release_metadata(self): + for flag, undo in ( + ("--assume-unchanged", "--no-assume-unchanged"), + ("--skip-worktree", "--no-skip-worktree"), + ): + for repo, relative in ( + (self.meta, "packages/govoplan-meta/pyproject.toml"), + (self.core, "pyproject.toml"), + ): + with self.subTest(flag=flag, repo=repo.name): + target = repo / relative + original = target.read_text() + git(repo, "update-index", flag, relative) + target.write_text( + original + + "\n# Hidden working-tree input differs from frozen HEAD\n" + ) + try: + self.assertEqual("", git_text(repo, "status", "--porcelain")) + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn( + "index entries", result["repositories"][0]["detail"] + ) + self.assert_no_tags() + finally: + target.write_text(original) + git(repo, "update-index", undo, relative) + + def test_read_only_git_target_is_not_repaired_or_tagged(self): + metadata = self.meta / ".git" + original = metadata.stat().st_mode & 0o7777 + metadata.chmod(0o500) + try: + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertEqual(0o500, metadata.stat().st_mode & 0o7777) + self.assert_no_tags() + finally: + metadata.chmod(original) + + def test_git_object_alternates_are_rejected_before_remote_lookup(self): + (self.meta / ".git/objects/info/alternates").write_text( + str(self.root / "outside-objects") + "\n" + ) + with patch.object( + meta_source_tag, + "registered_source_origin_issues", + side_effect=AssertionError("must reject alternates before Git"), + ): + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("alternates", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_owned_worktree_metadata_inside_private_workspace_is_supported(self): + main_checkout = self.workspace / "meta-main-storage" + self.meta.rename(main_checkout) + git(main_checkout, "worktree", "add", "--force", str(self.meta), "main") + self.assertTrue((self.meta / ".git").is_file()) + result = self.tag(apply=True) + self.assertEqual("tagged", result["status"], result) + filesystem = result["source_receipts"]["govoplan"]["filesystem"] + self.assertEqual( + str(main_checkout / ".git"), filesystem["git_common_directory"][0] + ) + + def test_git_directory_replacement_with_same_head_changes_frozen_receipt(self): + preview = meta_source_tag._preview_repositories + + def swapped_git_directory(**kwargs): + result = preview(**kwargs) + original = self.meta / ".git" + backup = self.root / "original-meta-git" + original.rename(backup) + shutil.copytree(backup, original) + return result + + with patch.object( + meta_source_tag, + "_preview_repositories", + side_effect=swapped_git_directory, + ): + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("receipt changed", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_non_main_and_divergent_live_main_fail_even_with_stale_tracking(self): + git(self.meta, "switch", "-c", "feature") + self.assertEqual("blocked", self.tag(apply=True)["status"]) + git(self.meta, "switch", "main") + clone = self.root / "other-writer" + git(self.root, "clone", "--branch", "main", str(self.meta_remote), str(clone)) + git(clone, "config", "user.name", "Other synthetic writer") + git(clone, "config", "user.email", "other@example.invalid") + (clone / "other.txt").write_text("remote divergence\n") + git(clone, "add", ".") + git(clone, "commit", "-m", "Remote main advanced") + git(clone, "push", "origin", "main") + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("live origin/main", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_symlink_checkout_is_not_a_registered_source(self): + original = self.workspace / "moved-meta" + self.meta.rename(original) + self.meta.symlink_to(original, target_is_directory=True) + result = self.tag(apply=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("symlink", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_lightweight_and_conflicting_annotated_tags_block(self): + git(self.meta, "tag", "v0.1.10") + self.assertEqual("blocked", self.tag(apply=True)["status"]) + git(self.meta, "tag", "-d", "v0.1.10") + git(self.meta, "tag", "-a", "v0.1.10", "-m", "First annotation") + git(self.meta, "push", "origin", "refs/tags/v0.1.10") + git(self.meta, "tag", "-d", "v0.1.10") + git(self.meta, "tag", "-a", "v0.1.10", "-m", "Different annotation") + self.assertEqual("blocked", self.tag(apply=True, push=True)["status"]) + self.assertFalse(ref_exists(self.core, "refs/tags/v0.1.10")) + + def test_meta_source_receipt_changed_after_preflight_blocks_before_first_effect( + self, + ): + preview = meta_source_tag._preview_repositories + + def changed_after_preflight(**kwargs): + result = preview(**kwargs) + (self.meta / "new-review.txt").write_text("changed after preflight\n") + self.commit_meta() + return result + + with patch.object( + meta_source_tag, + "_preview_repositories", + side_effect=changed_after_preflight, + ): + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("receipt changed", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_fabricated_push_success_without_remote_receipt_fails_and_stops_batch(self): + original = meta_source_tag.run + + def run(command, **kwargs): + if command[:3] == ("git", "push", "--atomic"): + return subprocess.CompletedProcess(command, 0, "", "") + return original(command, **kwargs) + + with patch.object(meta_source_tag, "run", side_effect=run): + result = self.tag(apply=True, push=True) + self.assertEqual("partial", result["status"], result) + self.assertEqual("failed", result["repositories"][0]["status"]) + self.assertEqual("skipped", result["repositories"][1]["status"]) + self.assertFalse(ref_exists(self.meta, "refs/tags/v0.1.10")) + self.assertFalse(ref_exists(self.core_remote, "refs/tags/v0.1.10")) + + def test_remote_tag_without_expected_main_receipt_is_not_success(self): + (self.core / "reviewed-change.txt").write_text("release source changes\n") + git(self.core, "add", ".") + git(self.core, "commit", "-m", "Advance reviewed Core source") + original = meta_source_tag.run + pushes = [] + + def tag_only(command, **kwargs): + if command[:3] == ("git", "push", "--atomic"): + pushes.append(command) + # Simulate a defective transport that claims atomic success, + # while publishing only the exact expected annotation object. + return original(("git", "push", "origin", command[-1]), **kwargs) + return original(command, **kwargs) + + with patch.object(meta_source_tag, "run", side_effect=tag_only): + result = self.tag(apply=True, push=True) + self.assertEqual("partial", result["status"], result) + self.assertEqual(1, len(pushes)) + self.assertIn("receipt changed", result["repositories"][0]["detail"]) + self.assertEqual( + git_text(self.core, "rev-parse", "refs/tags/v0.1.10"), + git_text(self.core_remote, "rev-parse", "refs/tags/v0.1.10"), + ) + self.assertNotEqual( + git_text(self.core, "rev-parse", "HEAD"), + git_text(self.core_remote, "rev-parse", "refs/heads/main"), + ) + self.assertFalse(ref_exists(self.meta, "refs/tags/v0.1.10")) + + def test_meta_receipt_is_rechecked_after_an_earlier_successful_publication(self): + original = meta_source_tag.run + + def changed_after_core(command, **kwargs): + result = original(command, **kwargs) + if ( + command[:3] == ("git", "push", "--atomic") + and kwargs["cwd"] == self.core + ): + (self.meta / "changed-review.txt").write_text( + "new Meta source after Core publication\n" + ) + self.commit_meta() + return result + + with patch.object(meta_source_tag, "run", side_effect=changed_after_core): + result = self.tag(apply=True, push=True) + self.assertEqual("partial", result["status"], result) + self.assertTrue(ref_exists(self.core_remote, "refs/tags/v0.1.10")) + self.assertFalse(ref_exists(self.meta, "refs/tags/v0.1.10")) + self.assertEqual("skipped", result["repositories"][1]["status"]) + + def test_unknown_selected_repository_cannot_use_meta_support_exception(self): + result = self.tag(repos=("govoplan", "unknown"), apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("not registered", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_selected_checkout_generator_is_never_executed(self): + malicious = self.meta / "tools/release/generate-developer-meta-package.py" + malicious.parent.mkdir(parents=True) + malicious.write_text( + 'raise RuntimeError("selected checkout must not execute")\n' + ) + self.commit_meta() + self.assertEqual("planned", self.tag()["status"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_repository_tag.py b/tests/test_release_repository_tag.py index e2a4642..434fab0 100644 --- a/tests/test_release_repository_tag.py +++ b/tests/test_release_repository_tag.py @@ -6,6 +6,7 @@ import subprocess import sys import tempfile import unittest +from unittest.mock import patch from pathlib import Path from fastapi.testclient import TestClient @@ -17,6 +18,7 @@ if str(RELEASE_ROOT) not in sys.path: sys.path.insert(0, str(RELEASE_ROOT)) from govoplan_release.repository_tag import tag_repositories # noqa: E402 +from govoplan_release import workspace as release_workspace # noqa: E402 from server.app import create_app # noqa: E402 @@ -39,6 +41,14 @@ class ReleaseRepositoryTagTests(unittest.TestCase): version="0.1.10", ) add_scoped_workflow_manifest(self.manifest_repo) + # The operator's test catalog explicitly registers known synthetic + # endpoints; production trust checks are not patched or bypassed. + self.registry = self.root / "registered-test-repositories.json" + self.registered = json.loads((META_ROOT / "repositories.json").read_text()) + for spec in self.registered["repositories"]: + spec["remote"] = str(self.root / f"{spec['name']}.git") + self.registry.write_text(json.dumps(self.registered)) + self.enterContext(patch.object(release_workspace, "REPOSITORIES_FILE", self.registry)) def tearDown(self) -> None: self.temporary.cleanup() @@ -128,6 +138,10 @@ class ReleaseRepositoryTagTests(unittest.TestCase): name="govoplan-core", version="0.1.10", ) + for spec in self.registered["repositories"]: + if spec["name"] == "govoplan-core": + spec["remote"] = str(remote_root / "govoplan-core.git") + self.registry.write_text(json.dumps(self.registered)) result = tag_repositories( repos=("govoplan-core",), diff --git a/tests/test_release_source_tag_batch.py b/tests/test_release_source_tag_batch.py new file mode 100644 index 0000000..0d110b7 --- /dev/null +++ b/tests/test_release_source_tag_batch.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools/release")) + +from govoplan_release import source_tag_batch # noqa: E402 +from govoplan_release.repository_tag import tag_repositories # noqa: E402 +import test_release_meta_source_tag as meta_fixture # noqa: E402 +import test_release_repository_tag as release_fixture # noqa: E402 +from test_release_repository_tag import git, git_text, ref_exists # noqa: E402 + + +class RegisteredSourceTagBatchTests(unittest.TestCase): + setUp = meta_fixture.MetaSourceTagTests.setUp + synchronize = meta_fixture.MetaSourceTagTests.synchronize + assert_no_tags = meta_fixture.MetaSourceTagTests.assert_no_tags + + def tag(self, *, repos=("govoplan-access",), apply=False, push=False): + return tag_repositories( + repos=repos, + repo_versions={repo: self.version for repo in repos}, + workspace_root=self.workspace, + apply=apply, + push=push, + ) + + def test_python_only_module_publishes_without_meta_or_core_checkout(self): + self.meta.rename(self.root / "unused-meta") + self.core.rename(self.root / "unused-core") + preview = self.tag(push=True) + self.assertEqual("planned", preview["status"], preview) + self.assertEqual("registered-source-batch-v1", preview["source_contract"]) + self.assertEqual({}, preview["bundle_input_receipts"]) + self.assertFalse(ref_exists(self.access, "refs/tags/v0.1.10")) + published = self.tag(apply=True, push=True) + self.assertEqual("published", published["status"], published) + self.assertEqual(["govoplan-access"], list(published["source_receipts"])) + self.assertEqual( + git_text(self.access, "rev-parse", "HEAD"), + git_text(self.access_remote, "rev-parse", "refs/heads/main"), + ) + + def test_core_only_batch_has_no_meta_composition_requirement(self): + self.meta.rename(self.root / "unused-meta") + result = self.tag(repos=("govoplan-core",), apply=True, push=True) + self.assertEqual("published", result["status"], result) + self.assertEqual(["govoplan-core"], list(result["source_receipts"])) + + def test_backend_only_publication_never_reads_irrelevant_unsafe_core_json(self): + webui = self.core / "webui" + webui.mkdir() + (webui / "package.release.json").write_text("invalid unselected Core JSON") + (webui / "package-lock.release.json").symlink_to( + self.root / "not-a-core-release-lock" + ) + from govoplan_release import version_alignment + + original = version_alignment._json_object + + def read(path): + if path.is_relative_to(webui): + raise AssertionError( + "backend-only publication must not read unrelated Core JSON" + ) + return original(path) + + with patch.object(version_alignment, "_json_object", side_effect=read): + result = self.tag(apply=True, push=True) + self.assertEqual("published", result["status"], result) + self.assertEqual({}, result["bundle_input_receipts"]) + + def test_source_origin_parent_and_read_only_target_guards_apply_without_meta(self): + for problem in ("origin", "parent", "read_only"): + with self.subTest(problem=problem): + parent_mode = self.root.stat().st_mode & 0o7777 + git_directory = self.access / ".git" + git_mode = git_directory.stat().st_mode & 0o7777 + if problem == "origin": + git( + self.access, + "config", + "remote.origin.pushurl", + str(self.root / "unknown.git"), + ) + elif problem == "parent": + self.root.chmod(0o777) + else: + git_directory.chmod(0o500) + try: + for apply in (False, True): + result = self.tag(apply=apply, push=True) + self.assertEqual("blocked", result["status"], result) + self.assert_no_tags() + finally: + self.root.chmod(parent_mode) + git_directory.chmod(git_mode) + if problem == "origin": + git(self.access, "config", "--unset", "remote.origin.pushurl") + + def test_wrong_owner_symlink_and_hidden_index_are_rejected_without_meta(self): + config = self.access / ".git/config" + original_stat = Path.lstat + + def wrong_owner(path, *args, **kwargs): + observed = original_stat(path, *args, **kwargs) + if path == config: + fields = list(observed) + fields[4] = os.geteuid() + 1 + return os.stat_result(fields) + return observed + + with patch.object(Path, "lstat", new=wrong_owner): + self.assertEqual("blocked", self.tag(apply=True)["status"]) + moved = self.root / "moved-access" + self.access.rename(moved) + self.access.symlink_to(moved, target_is_directory=True) + try: + self.assertEqual("blocked", self.tag(apply=True)["status"]) + finally: + self.access.unlink() + moved.rename(self.access) + for flag, undo in ( + ("--assume-unchanged", "--no-assume-unchanged"), + ("--skip-worktree", "--no-skip-worktree"), + ): + with self.subTest(flag=flag): + git(self.access, "update-index", flag, "pyproject.toml") + try: + self.assertEqual("blocked", self.tag(apply=True)["status"]) + self.assert_no_tags() + finally: + git(self.access, "update-index", undo, "pyproject.toml") + + def test_live_remote_divergence_blocks_without_cached_tracking_update(self): + clone = self.root / "other-access-writer" + git(self.root, "clone", "--branch", "main", str(self.access_remote), str(clone)) + git(clone, "config", "user.name", "Synthetic writer") + git(clone, "config", "user.email", "writer@example.invalid") + (clone / "advance.txt").write_text("new remote main\n") + git(clone, "add", ".") + git(clone, "commit", "-m", "Advance remote without updating original tracking") + git(clone, "push", "origin", "main") + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("live origin/main", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_ignored_selected_version_metadata_cannot_supply_an_untagged_artifact(self): + project = self.access / "pyproject.toml" + original = project.read_text() + git(self.access, "rm", "pyproject.toml") + (self.access / ".gitignore").write_text("/pyproject.toml\n") + git(self.access, "add", ".gitignore") + git(self.access, "commit", "-m", "Ignored metadata absent from frozen tree") + project.write_text(original) + self.assertEqual("", git_text(self.access, "status", "--porcelain")) + result = self.tag(apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn("must be tracked", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_lightweight_and_different_annotation_objects_remain_immutable(self): + git(self.access, "tag", "v0.1.10") + self.assertEqual("blocked", self.tag(apply=True)["status"]) + git(self.access, "tag", "-d", "v0.1.10") + git(self.access, "tag", "-a", "v0.1.10", "-m", "Original immutable annotation") + git(self.access, "push", "origin", "refs/tags/v0.1.10") + original = git_text(self.access_remote, "rev-parse", "refs/tags/v0.1.10") + git(self.access, "tag", "-d", "v0.1.10") + git( + self.access, + "tag", + "-a", + "v0.1.10", + "-m", + "Conflicting immutable annotation", + ) + self.assertEqual("blocked", self.tag(apply=True, push=True)["status"]) + self.assertEqual( + original, git_text(self.access_remote, "rev-parse", "refs/tags/v0.1.10") + ) + + def test_changed_source_after_preflight_stops_before_first_batch_effect(self): + original = source_tag_batch._preview_repositories + + def changed(**kwargs): + result = original(**kwargs) + (self.access / "changed-source.txt").write_text( + "new source after preflight\n" + ) + git(self.access, "add", ".") + git(self.access, "commit", "-m", "Source changed") + return result + + with patch.object( + source_tag_batch, "_preview_repositories", side_effect=changed + ): + result = self.tag( + repos=("govoplan-core", "govoplan-access"), apply=True, push=True + ) + self.assertEqual("blocked", result["status"], result) + self.assertIn("receipt changed", result["repositories"][0]["detail"]) + self.assert_no_tags() + + def test_false_push_success_is_not_a_receipt_and_never_retries(self): + original = source_tag_batch.run + pushes = [] + + def run(command, **kwargs): + if command[:3] == ("git", "push", "--atomic"): + pushes.append(command) + return subprocess.CompletedProcess(command, 0, "", "") + return original(command, **kwargs) + + with patch.object(source_tag_batch, "run", side_effect=run): + result = self.tag( + repos=("govoplan-access", "govoplan-core"), apply=True, push=True + ) + self.assertEqual("partial", result["status"], result) + self.assertEqual(1, len(pushes)) + self.assertEqual("skipped", result["repositories"][1]["status"]) + self.assertFalse(ref_exists(self.core, "refs/tags/v0.1.10")) + self.assertFalse(ref_exists(self.access_remote, "refs/tags/v0.1.10")) + + def _ready_bundle(self): + self.repo = self.core # Existing fixture's Core name. + campaign, remote = ( + release_fixture.ReleaseRepositoryTagTests._staged_campaign_bundle(self) + ) + self.specs.append( + { + "name": "govoplan-campaign", + "category": "module", + "subtype": "domain", + "path": "govoplan-campaign", + "remote": str(remote), + } + ) + self.registry.write_text(json.dumps({"repositories": self.specs})) + self.assertEqual( + "tagged", self.tag(repos=("govoplan-campaign",), apply=True)["status"] + ) + lock_path = self.core / "webui/package-lock.release.json" + payload = json.loads(lock_path.read_text()) + package = payload["packages"]["node_modules/@govoplan/campaign-webui"] + package["version"] = self.version + package["resolved"] = ( + "git+ssh://git@example.test/acme/govoplan-campaign.git#" + + git_text(campaign, "rev-parse", "HEAD") + ) + lock_path.write_text(json.dumps(payload)) + # Core is a reviewed input only here: do not require an unrelated tag + # or silently impose a new clean-Core prerequisite for module release. + return campaign, remote, lock_path + + def test_module_publication_freezes_core_inputs_without_requiring_core_tag(self): + _campaign, _remote, _lock = self._ready_bundle() + self.assertFalse(ref_exists(self.core, "refs/tags/v0.1.10")) + result = self.tag(repos=("govoplan-campaign",), apply=True, push=True) + self.assertEqual("published", result["status"], result) + self.assertEqual(["govoplan-campaign"], list(result["source_receipts"])) + self.assertEqual( + {"webui/package.release.json", "webui/package-lock.release.json"}, + set(result["bundle_input_receipts"]), + ) + self.assertFalse(ref_exists(self.core, "refs/tags/v0.1.10")) + + def test_changed_or_group_writable_core_bundle_inputs_block_module_publication( + self, + ): + campaign, remote, lock = self._ready_bundle() + original = source_tag_batch._preview_repositories + + def changed(**kwargs): + result = original(**kwargs) + lock.write_text(lock.read_text() + "\n") + return result + + with patch.object( + source_tag_batch, "_preview_repositories", side_effect=changed + ): + result = self.tag(repos=("govoplan-campaign",), apply=True, push=True) + self.assertEqual("blocked", result["status"], result) + self.assertIn( + "bundle input receipt changed", result["repositories"][0]["detail"] + ) + self.assertFalse(ref_exists(remote, "refs/tags/v0.1.10")) + lock.chmod(0o666) + try: + self.assertEqual( + "blocked", + self.tag(repos=("govoplan-campaign",), apply=True, push=True)["status"], + ) + finally: + lock.chmod(0o644) + self.assertTrue(ref_exists(campaign, "refs/tags/v0.1.10")) + self.assertFalse(ref_exists(remote, "refs/tags/v0.1.10")) + + def test_read_only_shared_preflight_has_no_apply_or_mutating_legacy_entry(self): + import inspect + from govoplan_release import repository_tag + + self.assertNotIn( + "apply", inspect.signature(repository_tag._preview_repositories).parameters + ) + self.assertFalse(hasattr(repository_tag, "_tag_repositories_legacy")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/checks/check-focused.sh b/tools/checks/check-focused.sh index 91dcc67..42323c6 100644 --- a/tools/checks/check-focused.sh +++ b/tools/checks/check-focused.sh @@ -51,6 +51,8 @@ cd "$META_ROOT" "$PYTHON" -m unittest tests.test_webui_package_facades "$PYTHON" -m unittest tests.test_module_package_workflows tests.test_package_registry_release "$PYTHON" -m unittest tests.test_deployment_installer tests.test_webui_release_dependency_retries +"$PYTHON" -m pytest -q tests/test_release_meta_source_tag.py tests/test_release_source_tag_batch.py tests/test_release_meta_preparation.py +"$PYTHON" -m unittest tests.test_isolated_work_composition "$PYTHON" -m unittest tests.test_capability_fit_evidence "$PYTHON" -m unittest tests.test_capability_fit_generation tests.test_capability_fit_review "$PYTHON" tools/assessments/generate-capability-fit-report.py --check @@ -101,6 +103,7 @@ PY "$PYTHON" "$META_ROOT/tools/checks/check-shared-webui-primitives.py" "$PYTHON" "$META_ROOT/tools/checks/check-shared-webui-foundations.py" "$PYTHON" -m unittest tests.test_module_system +"$PYTHON" -m unittest tests.test_bounded_process "$PYTHON" -m unittest tests.test_ownership_history_migration tests.test_ownership tests.test_ownership_api "$PYTHON" -m unittest tests.test_navigation_preferences tests.test_api_smoke.ApiSmokeTests.test_navigation_separator_layout_survives_system_tenant_and_personal_saves "$PYTHON" -m pytest -q \ @@ -109,7 +112,13 @@ PY /mnt/DATA/git/govoplan-files/tests/test_archive_staging.py \ /mnt/DATA/git/govoplan-files/tests/test_upload_response_batching.py \ /mnt/DATA/git/govoplan-files/tests/test_archive_performance.py +"$PYTHON" -m pytest -q \ + /mnt/DATA/git/govoplan-files/tests/test_archive_workers.py \ + /mnt/DATA/git/govoplan-files/tests/test_archive_inspection_bounds.py \ + /mnt/DATA/git/govoplan-files/tests/test_archives.py "$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-access/tests/test_external_function_mapping_migration.py +"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-access/tests +"$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-templates/tests "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-connectors/tests "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-datasources/tests "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-dataflow/tests @@ -151,9 +160,14 @@ PY "$PYTHON" -m pytest -q /mnt/DATA/git/govoplan-mail/tests/test_campaign_protocol_authorization.py /mnt/DATA/git/govoplan-mail/tests/test_campaign_imap_batch.py "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests "$PYTHON" -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count +"$PYTHON" -m unittest \ + tests.test_api_smoke.ApiSmokeTests.test_managed_attachment_patterns_preview_build_and_mock_send \ + tests.test_api_smoke.ApiSmokeTests.test_reports_and_job_review_are_scoped_to_the_selected_version \ + tests.test_api_smoke.ApiSmokeTests.test_worker_loss_becomes_unknown_and_requires_reconciliation_before_retry cd "$ROOT/webui" "$NPM" run test:api-client-cache +"$NPM" run test:auth-action-state "$NPM" run test:dependency-security "$NPM" run test:layout-primitives "$NPM" run test:mail-components @@ -161,6 +175,9 @@ cd "$ROOT/webui" "$NPM" run test:module-permutations "$NPM" run test:conformance +cd /mnt/DATA/git/govoplan-access/webui +"$NPM" run test:passwords + "$WEBUI_BIN/tsc" -p /mnt/DATA/git/govoplan-payments/webui/tsconfig.json cd /mnt/DATA/git/govoplan-payments/webui diff --git a/tools/deployment/govoplan_deploy/model.py b/tools/deployment/govoplan_deploy/model.py index b631244..6ddb9c0 100644 --- a/tools/deployment/govoplan_deploy/model.py +++ b/tools/deployment/govoplan_deploy/model.py @@ -13,7 +13,10 @@ from urllib.parse import urlsplit SCHEMA_VERSION = 1 DEFAULT_GARAGE_IMAGE = "dxflrs/garage:v2.3.0" -DEFAULT_LOAD_BALANCER_IMAGE = "haproxy:3.2.21-alpine" +DEFAULT_LOAD_BALANCER_IMAGE = ( + "haproxy:3.2.23-alpine@sha256:" + "6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e" +) DEFAULT_INGRESS_IMAGE = "caddy:2.10.2-alpine" INSTALLATION_ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,47}$") ENV_NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$") diff --git a/tools/inventory/endpoint-surface-declarations.json b/tools/inventory/endpoint-surface-declarations.json index 2464ea8..cb89ce0 100644 --- a/tools/inventory/endpoint-surface-declarations.json +++ b/tools/inventory/endpoint-surface-declarations.json @@ -2133,6 +2133,34 @@ "rationale": "This capability-first module intentionally exposes a headless API for other modules and integrations.", "repository": "govoplan-workflow-engine" }, + { + "category": "intentionally_headless", + "method": "GET", + "path": "/workflow/instances/summaries", + "rationale": "Workflow Engine publishes bounded, current-authorized summary discovery for module and API consumers; the existing Workflow UI retains its compatible full-history contract. See owning topic workflow.instance-history.", + "repository": "govoplan-workflow-engine" + }, + { + "category": "intentionally_headless", + "method": "GET", + "path": "/workflow/instances/{}/events", + "rationale": "Workflow Engine publishes current-authorized, sequence-bounded event history pages with explicit total and continuation semantics for module and API consumers. See owning topic workflow.instance-history.", + "repository": "govoplan-workflow-engine" + }, + { + "category": "intentionally_headless", + "method": "GET", + "path": "/workflow/instances/{}/steps", + "rationale": "Workflow Engine publishes current-authorized, sequence-bounded step history pages with explicit total and continuation semantics for module and API consumers. See owning topic workflow.instance-history.", + "repository": "govoplan-workflow-engine" + }, + { + "category": "intentionally_headless", + "method": "GET", + "path": "/workflow/instances/{}/summary", + "rationale": "Workflow Engine publishes a current-authorized, history-free instance summary for module and API consumers; the existing Workflow UI retains its compatible full-history detail contract. See owning topic workflow.instance-history.", + "repository": "govoplan-workflow-engine" + }, { "category": "ui_reachable", "method": "POST", diff --git a/tools/release/govoplan_release/git_state.py b/tools/release/govoplan_release/git_state.py index 960f004..bdb75e6 100644 --- a/tools/release/govoplan_release/git_state.py +++ b/tools/release/govoplan_release/git_state.py @@ -11,7 +11,7 @@ import tomllib from .contracts import parse_manifest_contract from .model import RepositorySnapshot, RepositorySpec, VersionSnapshot -from .workspace import resolve_repo_path +from .workspace import load_repository_specs, resolve_repo_path def collect_repository_snapshot( @@ -98,6 +98,7 @@ def collect_repository_snapshot( def collect_versions(path: Path) -> VersionSnapshot: return VersionSnapshot( pyproject=read_pyproject_version(path), + developer_meta=read_developer_meta_version(path), package=read_json_version(path / "package.json"), webui_package=read_json_version(path / "webui" / "package.json"), manifests=read_manifest_versions(path), @@ -105,6 +106,35 @@ def collect_versions(path: Path) -> VersionSnapshot: ) +def registered_developer_meta_path(path: Path) -> Path | None: + """Recognize only the catalog's explicit Meta support-repository identity. + + This is metadata discovery, not authorization to access a remote or mutate + a checkout. Tagging applies its separate registered source trust contract. + """ + for spec in load_repository_specs(include_website=False): + if ( + spec.name == "govoplan" + and spec.category == "system" + and spec.subtype == "meta" + and path.absolute() == resolve_repo_path(spec, path.parent).absolute() + ): + return path / "packages" / "govoplan-meta" / "pyproject.toml" + return None + + +def read_developer_meta_version(path: Path) -> str | None: + package = registered_developer_meta_path(path) + if package is None or not package.is_file(): + return None + with package.open("rb") as handle: + project = tomllib.load(handle).get("project") + if isinstance(project, dict) and project.get("name") == "govoplan": + version = project.get("version") + return version if isinstance(version, str) else None + return None + + def read_pyproject_version(path: Path) -> str | None: pyproject = path / "pyproject.toml" if not pyproject.exists(): diff --git a/tools/release/govoplan_release/meta_preparation.py b/tools/release/govoplan_release/meta_preparation.py new file mode 100644 index 0000000..57893cd --- /dev/null +++ b/tools/release/govoplan_release/meta_preparation.py @@ -0,0 +1,380 @@ +"""Receipt-bound, out-of-run preparation of the real developer meta-package. + +This deliberately does not commit, tag, publish, or update a running release +console. The complete generated file is reviewed in a separate source checkout. +""" + +from __future__ import annotations + +import copy +import hashlib +import os +from pathlib import Path +import runpy +import stat +import tempfile +import tomllib + +from .git_state import ( + collect_repository_snapshot, + git, + git_text, + registered_developer_meta_path, +) +from .repository_tag import normalize_version, remote_tag_commit, run +from .source_provenance import registered_source_origin_issues +from .source_tag_batch import ( + _OBJECT, + _owned_path, + _source_filesystem, + _tracked_worktree, + _trusted_ancestry, +) +from .workspace import META_ROOT, load_repository_specs, resolve_repo_path + +PACKAGE = "packages/govoplan-meta/pyproject.toml" +MAX_INPUT_FILES = 128 +MAX_INPUT_BYTES = 2 * 1024 * 1024 +MAX_TOTAL_BYTES = 16 * 1024 * 1024 +GENERATOR = ".operator/generate-developer-meta-package.py" + + +class MetaPreparationError(ValueError): + """Preparation is blocked, or an applied file needs explicit reconciliation.""" + + +class MetaPreparationAmbiguous(MetaPreparationError): + """The file effect may have happened and requires explicit reconciliation.""" + + +def preparation_command(*, workspace: Path, target_version: str) -> str: + import shlex + + return " ".join( + shlex.quote(value) + for value in ( + "python", + str(META_ROOT / "tools/release/prepare-developer-meta-package.py"), + "--workspace", + str(workspace), + "--target-version", + target_version, + ) + ) + + +def _read_input(path: Path) -> tuple[bytes, dict]: + def identity(value): + return ( + value.st_dev, + value.st_ino, + value.st_uid, + value.st_gid, + value.st_mode, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + before = path.lstat() + if ( + not stat.S_ISREG(before.st_mode) + or before.st_uid != os.geteuid() + or before.st_mode & 0o022 + or not 0 < before.st_size <= MAX_INPUT_BYTES + ): + raise MetaPreparationError( + "Preparation inputs must be owned, bounded regular files." + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as source: + opened = os.fstat(source.fileno()) + if identity(opened) != identity(before): + raise MetaPreparationError("Preparation input changed before reading.") + payload = source.read(before.st_size + 1) + if ( + identity(os.fstat(source.fileno())) != identity(opened) + or len(payload) != before.st_size + ): + raise MetaPreparationError("Preparation input changed while reading.") + return payload, { + "sha256": hashlib.sha256(payload).hexdigest(), + "identity": [ + before.st_dev, + before.st_ino, + before.st_uid, + before.st_gid, + stat.S_IMODE(before.st_mode), + before.st_size, + ], + } + + +def _snapshot(*, repo_path: Path, target_version: str, output_dirty: bool = False): + workspace = repo_path.parent + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + if registered_developer_meta_path(repo_path) != repo_path / PACKAGE: + raise MetaPreparationError( + "Only the registered Meta nested-package identity can be prepared." + ) + if repo_path.resolve() == META_ROOT.resolve(): + raise MetaPreparationError( + "Prepare a separate registered source checkout, never the running operator tooling." + ) + paths = [repo_path / PACKAGE, repo_path / "requirements-release.txt"] + for path in workspace.glob("govoplan-*/pyproject.toml"): + paths.append(path) + if len(paths) > MAX_INPUT_FILES: + raise MetaPreparationError( + "Developer composition exceeds its input-file bound." + ) + if workspace / "govoplan-core/pyproject.toml" not in paths: + raise MetaPreparationError( + "Prepare and commit matching Core metadata before Meta preparation." + ) + repositories = {"govoplan": repo_path} + for path in paths[2:]: + name = path.parent.name + if ( + name not in specs + or resolve_repo_path(specs[name], workspace) != path.parent + ): + raise MetaPreparationError( + "Developer composition contains an unregistered package checkout." + ) + repositories[name] = path.parent + filesystems = {} + for name, path in repositories.items(): + filesystems[name] = _source_filesystem(path=path, workspace=workspace) + _tracked_worktree(path) + issues = registered_source_origin_issues( + repo_versions={name: target_version for name in repositories}, + workspace=workspace, + remote="origin", + ) + if issues: + raise MetaPreparationError( + "Preparation source origins do not match the registered repositories." + ) + sources = {} + for name, path in sorted(repositories.items()): + snapshot = collect_repository_snapshot( + specs[name], + workspace_root=workspace, + target_tag=None, + online=False, + ) + dirty_allowed = ( + output_dirty + and name == "govoplan" + and snapshot.dirty_entries == (f" M {PACKAGE}",) + ) + if ( + snapshot.errors + or not snapshot.has_head + or snapshot.branch != "main" + or snapshot.upstream != "origin/main" + or snapshot.behind + or (snapshot.dirty and not dirty_allowed) + ): + raise MetaPreparationError( + "Preparation requires reviewed clean main sources tracking origin/main." + ) + head = git_text(path, "rev-parse", "--verify", "HEAD") + live = run( + ("git", "ls-remote", "--exit-code", "--heads", "origin", "refs/heads/main"), + cwd=path, + ) + lines = live.stdout.strip().splitlines() + if live.returncode or len(lines) != 1: + raise MetaPreparationError("Could not verify live preparation source main.") + remote_main, separator, reference = lines[0].partition("\t") + if ( + not _OBJECT.fullmatch(head) + or not _OBJECT.fullmatch(remote_main) + or not separator + or reference != "refs/heads/main" + or git(path, "merge-base", "--is-ancestor", remote_main, head).returncode + ): + raise MetaPreparationError( + "Preparation source main diverged; fetch and review separately." + ) + sources[name] = { + "head": head, + "remote_main": remote_main, + "filesystem": filesystems[name], + } + tag = f"v{target_version}" + published = remote_tag_commit(repo_path, remote="origin", tag=tag) + if ( + published.error + or published.tag_object + or git_text(repo_path, "rev-parse", "--verify", f"refs/tags/{tag}") + ): + raise MetaPreparationError( + "An existing or unverifiable target Meta tag blocks source preparation." + ) + inputs, payloads = {}, {} + total = 0 + for path in sorted(paths): + payload, identity = _read_input(path) + total += len(payload) + if total > MAX_TOTAL_BYTES: + raise MetaPreparationError( + "Developer composition exceeds its aggregate input bound." + ) + relative = path.relative_to(workspace).as_posix() + inputs[relative], payloads[relative] = identity, payload + generator_path = META_ROOT / "tools/release/generate-developer-meta-package.py" + _trusted_ancestry(generator_path.parent) + _owned_path(META_ROOT, directory=True) + generator, generator_identity = _read_input(generator_path) + if total + len(generator) > MAX_TOTAL_BYTES: + raise MetaPreparationError( + "Developer composition exceeds its aggregate input bound." + ) + payloads[GENERATOR] = generator + current = tomllib.loads(payloads[f"govoplan/{PACKAGE}"].decode("utf-8")).get( + "project", {} + ) + core = tomllib.loads(payloads["govoplan-core/pyproject.toml"].decode("utf-8")).get( + "project", {} + ) + if ( + not isinstance(current, dict) + or current.get("name") != "govoplan" + or not isinstance(current.get("version"), str) + ): + raise MetaPreparationError( + "Nested developer-package identity and version must be exact." + ) + if ( + not isinstance(core, dict) + or core.get("name") != "govoplan-core" + or core.get("version") != target_version + ): + raise MetaPreparationError( + "Prepare and commit Core at the requested target version before Meta preparation." + ) + from .version_alignment import repository_version_issues + + if repository_version_issues( + workspace / "govoplan-core", expected_version=target_version + ): + raise MetaPreparationError( + "Prepare aligned Core version metadata before Meta preparation." + ) + receipt = { + "kind": "developer_meta_preparation_v1", + "workspace": str(workspace), + "target_version": target_version, + "sources": sources, + "inputs": inputs, + "operator_generator": generator_identity, + } + return receipt, payloads + + +def _render(payloads: dict[str, bytes]) -> bytes: + # The trusted operator generator sees only the frozen bounded data snapshot. + with tempfile.TemporaryDirectory(prefix="govoplan-meta-render-") as temporary: + workspace = Path(temporary) + for relative, payload in payloads.items(): + path = workspace / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + render = runpy.run_path(str(workspace / GENERATOR))["render"] + result = render( + workspace=workspace, + requirements=workspace / "govoplan/requirements-release.txt", + ).encode("utf-8") + if len(result) > MAX_INPUT_BYTES: + raise MetaPreparationError( + "Generated developer package exceeds its output bound." + ) + return result + + +def preview_meta_mutation(*, repo_path: Path, target_version: str): + version = normalize_version(target_version) + if not version: + raise MetaPreparationError("A valid target release version is required.") + receipt, payloads = _snapshot(repo_path=repo_path, target_version=version) + after = _render(payloads) + observed, _ = _snapshot(repo_path=repo_path, target_version=version) + if observed != receipt: + raise MetaPreparationError( + "Developer preparation inputs changed during preview." + ) + receipt["output_sha256"] = hashlib.sha256(after).hexdigest() + return receipt, payloads[f"govoplan/{PACKAGE}"], after + + +def prepare_developer_meta_package( + *, + repo_path: Path, + target_version: str, + apply: bool = False, + expected_receipt=None, + confirm_out_of_run: bool = False, +) -> dict: + """Preview or explicitly apply one full generated file; never commit/publish.""" + try: + receipt, before, after = preview_meta_mutation( + repo_path=repo_path, target_version=target_version + ) + result = { + "status": "planned" if before != after else "noop", + "path": PACKAGE, + "receipt": receipt, + "after_sha256": hashlib.sha256(after).hexdigest(), + "changed": before != after, + } + if not apply: + return result + if not confirm_out_of_run: + raise MetaPreparationError( + "Explicitly confirm that no durable run is active for this source workspace." + ) + if receipt != expected_receipt: + raise MetaPreparationError( + "Preparation inputs changed since the reviewed preview; create a fresh preview." + ) + if before == after: + return result + from .version_metadata import _atomic_write + + source_receipt = { + key: value for key, value in receipt.items() if key != "output_sha256" + } + observed, _ = _snapshot( + repo_path=repo_path, target_version=receipt["target_version"] + ) + if observed != source_receipt: + raise MetaPreparationError( + "Preparation inputs changed before the file effect; create a fresh preview." + ) + try: + _atomic_write(repo_path / PACKAGE, after) + observed, payloads = _snapshot( + repo_path=repo_path, + target_version=receipt["target_version"], + output_dirty=True, + ) + comparison = copy.deepcopy(observed) + output = f"govoplan/{PACKAGE}" + comparison["inputs"][output] = receipt["inputs"][output] + if comparison != source_receipt or payloads[output] != after: + raise MetaPreparationError("Preparation inputs changed after writing.") + except Exception as exc: + raise MetaPreparationAmbiguous( + "Generated file may have been written but its write/post-check failed; " + "review the delta and reconcile manually." + ) from exc + return {**result, "status": "prepared", "after_receipt": observed} + except MetaPreparationError: + raise + except (OSError, UnicodeError, ValueError, KeyError, TypeError) as exc: + raise MetaPreparationError( + f"Developer preparation failed closed ({type(exc).__name__})." + ) from exc diff --git a/tools/release/govoplan_release/model.py b/tools/release/govoplan_release/model.py index 5e3bac4..89cf154 100644 --- a/tools/release/govoplan_release/model.py +++ b/tools/release/govoplan_release/model.py @@ -23,6 +23,7 @@ class RepositorySpec: @dataclass(frozen=True, slots=True) class VersionSnapshot: pyproject: str | None = None + developer_meta: str | None = None package: str | None = None webui_package: str | None = None manifests: tuple[str, ...] = () @@ -32,6 +33,7 @@ class VersionSnapshot: def primary(self) -> str | None: return ( self.pyproject + or self.developer_meta or self.package or self.webui_package or (self.manifests[0] if self.manifests else None) diff --git a/tools/release/govoplan_release/repository_tag.py b/tools/release/govoplan_release/repository_tag.py index cd2e973..e1edc44 100644 --- a/tools/release/govoplan_release/repository_tag.py +++ b/tools/release/govoplan_release/repository_tag.py @@ -34,7 +34,24 @@ def tag_repositories( apply: bool = False, # noqa: A002 - mirrors API field. push: bool = False, ) -> dict[str, object]: - """Create annotated tags and optionally publish branch and tag atomically. + from .source_tag_batch import tag_source_batch + + return tag_source_batch( + repos=repos, repo_versions=repo_versions, workspace_root=workspace_root, + remote=remote, message=message, apply=apply, push=push, + ) + + +def _preview_repositories( + *, + repos: tuple[str, ...], + repo_versions: dict[str, str], + workspace_root: Path | str | None = None, + remote: str = "origin", + message: str | None = None, + push: bool = False, +) -> dict[str, object]: + """Read-only shared manifest/version/composition/tag preflight. A release tag is only created for a clean, aligned, non-behind worktree. Both local and remote tags are resolved to commits before mutation so an @@ -64,42 +81,7 @@ def tag_repositories( f"{issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})" ) - if apply: - preflight = tag_repositories( - repos=selected, - repo_versions=repo_versions, - workspace_root=workspace, - remote=remote, - message=message, - apply=False, - push=push, - ) - preflight_rows = preflight.get("repositories") - if isinstance(preflight_rows, list) and any( - isinstance(item, dict) and item.get("status") in {"blocked", "failed"} - for item in preflight_rows - ): - blocked_rows = [] - for item in preflight_rows: - if not isinstance(item, dict) or item.get("status") in {"blocked", "failed"}: - blocked_rows.append(item) - continue - blocked_rows.append( - { - **item, - "status": "skipped", - "detail": "preflight passed, but no release tag was changed because another selected repository is blocked", - } - ) - return { - "status": "blocked", - "apply": True, - "push": push, - "remote": remote, - "detail": "batch preflight failed; no selected repository was mutated", - "repositories": blocked_rows, - } - elif selected: + if selected: manifest_gate_issue = manifest_shape_gate_issue(workspace) if manifest_gate_issue: return { @@ -255,130 +237,17 @@ def tag_repositories( "remote_tag_object": remote_result.tag_object, } ) - if not apply: - detail = preview_detail(tag=tag, local_commit=local_commit, remote_commit=remote_result.commit, push=push) - status = "noop" if remote_result.commit or (local_commit and not push) else "planned" - results.append({**row, "status": status, "detail": detail}) - continue - - if remote_result.commit: - if not local_commit: - fetch_result = run(("git", "fetch", remote, f"refs/tags/{tag}:refs/tags/{tag}"), cwd=path) - if fetch_result.returncode != 0: - results.append( - { - **row, - "status": "failed", - "detail": f"remote tag {tag} exists at HEAD but could not be fetched locally", - "returncode": fetch_result.returncode, - "stdout": compact_output(fetch_result.stdout), - "stderr": compact_output(fetch_result.stderr), - } - ) - continue - results.append( - { - **row, - "status": "published", - "detail": f"immutable tag {tag} is already published at HEAD", - "after_local_tag_commit": head_commit, - "after_remote_tag_commit": head_commit, - } - ) - continue - - created = False - if not local_commit: - create_result = run(create_command, cwd=path) - if create_result.returncode != 0: - results.append( - { - **row, - "status": "failed", - "detail": f"could not create annotated tag {tag}", - "returncode": create_result.returncode, - "stdout": compact_output(create_result.stdout), - "stderr": compact_output(create_result.stderr), - } - ) - continue - created = True - - if not push: - results.append( - { - **row, - "status": "tagged" if created else "noop", - "detail": f"created annotated tag {tag} at HEAD" if created else f"annotated tag {tag} already exists at HEAD", - "after_local_tag_commit": head_commit, - } - ) - continue - - publish_result = run(publish_command, cwd=path) - if publish_result.returncode != 0: - results.append( - { - **row, - "status": "failed", - "detail": f"created local tag {tag}, but atomic branch and tag publication failed" if created else f"atomic branch and tag publication failed for {tag}", - "returncode": publish_result.returncode, - "after_local_tag_commit": head_commit, - "stdout": compact_output(publish_result.stdout), - "stderr": compact_output(publish_result.stderr), - } - ) - continue - after_local_object = git_text(path, "rev-parse", "--verify", f"refs/tags/{tag}") - after_remote = remote_tag_commit(path, remote=remote, tag=tag) - if ( - after_remote.error - or not after_remote.annotated - or after_remote.commit != head_commit - or after_remote.tag_object != after_local_object - ): - verification_detail = after_remote.error or "remote tag did not resolve to the published annotated tag object at HEAD" - results.append( - { - **row, - "status": "failed", - "detail": f"Git push returned success, but the remote release-tag postcondition failed: {verification_detail}", - "returncode": publish_result.returncode, - "after_local_tag_commit": head_commit, - "after_local_tag_object": after_local_object, - "after_remote_tag_commit": after_remote.commit, - "after_remote_tag_object": after_remote.tag_object, - "stdout": compact_output(publish_result.stdout), - "stderr": compact_output(publish_result.stderr), - } - ) - continue - results.append( - { - **row, - "status": "published", - "detail": f"published branch {snapshot.branch} and immutable tag {tag} atomically to {remote}", - "returncode": publish_result.returncode, - "after_local_tag_commit": head_commit, - "after_remote_tag_commit": head_commit, - "after_local_tag_object": after_local_object, - "after_remote_tag_object": after_remote.tag_object, - "stdout": compact_output(publish_result.stdout), - "stderr": compact_output(publish_result.stderr), - } - ) + detail = preview_detail(tag=tag, local_commit=local_commit, remote_commit=remote_result.commit, push=push) + row_status = "noop" if remote_result.commit or (local_commit and not push) else "planned" + results.append({**row, "status": row_status, "detail": detail}) if any(item["status"] in {"blocked", "failed"} for item in results): - status = "blocked" if not apply else "partial" - elif any(item["status"] == "published" for item in results): - status = "published" - elif any(item["status"] == "tagged" for item in results): - status = "tagged" + result_status = "blocked" elif any(item["status"] == "planned" for item in results): - status = "planned" + result_status = "planned" else: - status = "noop" - return {"status": status, "apply": apply, "push": push, "remote": remote, "repositories": results} + result_status = "noop" + return {"status": result_status, "apply": False, "push": push, "remote": remote, "repositories": results} def normalize_version(value: str | None) -> str: diff --git a/tools/release/govoplan_release/selective_planner.py b/tools/release/govoplan_release/selective_planner.py index 868ec78..a42134c 100644 --- a/tools/release/govoplan_release/selective_planner.py +++ b/tools/release/govoplan_release/selective_planner.py @@ -8,6 +8,7 @@ from pathlib import Path import shlex from .contracts import validate_contracts +from .git_state import registered_developer_meta_path, read_pyproject_version from .model import ( CompatibilityIssue, InterfaceProviderSnapshot, @@ -111,6 +112,39 @@ def apply_repository_version_gate( version_update_supported_by_repo: dict[str, bool] = {} deferred_core_lock_repos: set[str] = set() for unit in units: + if registered_developer_meta_path(workspace / unit.repo) is not None: + from .meta_preparation import preparation_command + + try: + core_version = read_pyproject_version(workspace / "govoplan-core") + except (OSError, ValueError, TypeError): + core_version = None + core_ready = core_version == unit.target_version + issues_by_repo.setdefault(unit.repo, []).append( + ReleaseGateFinding( + code="developer_meta_out_of_run" if core_ready else "developer_meta_core_preparation_required", + severity="blocker", + message=( + "Meta is an out-of-run support release, not a self-updating durable executor." + if core_ready else + "Prepare and commit Core at the requested target before regenerating Meta." + ), + remediation=( + "Complete Core and module preparation first. Stop active durable runs for this workspace. " + "In a separate trusted source checkout, preview " + + preparation_command(workspace=workspace, target_version=unit.target_version) + + ". Review its receipt; apply with --receipt --apply --confirm-out-of-run. " + "Review and commit the whole generated package, publish the matching Core release first, " + "then use guarded Meta source tagging/publication and create a fresh durable run." + ), + repo=unit.repo, source="developer meta-package preparation", + expected=unit.target_version, actual=core_version or "missing Core version", + ) + ) + version_update_supported_by_repo[unit.repo] = False + # Its complete canonical composition remains a publication gate; + # this plan must not claim a generic in-run mutation/commit path. + continue version_update_supported = unit.current_version == unit.target_version if unit.current_version and unit.current_version != unit.target_version: try: @@ -437,6 +471,7 @@ def build_unit( value for value in ( repo.versions.pyproject, + repo.versions.developer_meta, repo.versions.package, repo.versions.webui_package, *repo.versions.manifests, @@ -572,7 +607,7 @@ def repository_capabilities( def dependency_ordered_units( units: tuple[ReleasePlanUnit, ...], ) -> tuple[ReleasePlanUnit, ...]: - """Order module providers before consumers while keeping Core last.""" + """Order modules before Core, followed by the out-of-run Meta support unit.""" by_repo = {unit.repo: unit for unit in units} providers: dict[str, set[str]] = {} @@ -592,8 +627,10 @@ def dependency_ordered_units( ) if "govoplan-core" in dependencies: dependencies["govoplan-core"].update( - repo for repo in by_repo if repo != "govoplan-core" + repo for repo in by_repo if repo not in {"govoplan-core", "govoplan"} ) + if "govoplan" in dependencies: + dependencies["govoplan"].update(repo for repo in by_repo if repo != "govoplan") ordered: list[ReleasePlanUnit] = [] remaining = set(by_repo) @@ -690,6 +727,8 @@ def dry_run_steps( *, units: tuple[ReleasePlanUnit, ...], dashboard: ReleaseDashboard, channel: str ) -> tuple[ReleasePlanStep, ...]: steps: list[ReleasePlanStep] = [] + meta_units = tuple(unit for unit in units if unit.repo == "govoplan") + units = tuple(unit for unit in units if unit.repo != "govoplan") snapshots = {repo.spec.name: repo for repo in dashboard.repositories} core_unit = next((unit for unit in units if unit.repo == "govoplan-core"), None) non_core_units = tuple(unit for unit in units if unit.repo != "govoplan-core") @@ -991,6 +1030,33 @@ def dry_run_steps( status="planned", ) ) + for unit in meta_units: + from .meta_preparation import preparation_command + + steps.extend(( + ReleasePlanStep( + id="govoplan:prepare-support", + title="Prepare the complete developer meta-package outside this run", + detail=( + "First prepare and commit Core at the target and review module/requirements inputs. " + "Stop active runs, preview and explicitly apply the frozen composition in a separate " + "source checkout; review and commit manually. No durable self-update is supported." + ), + command=preparation_command(workspace=Path(dashboard.workspace_root), target_version=unit.target_version), + cwd=dashboard.meta_root, repo=unit.repo, status="needs-executor", + ), + ReleasePlanStep( + id="govoplan:publish-support", + title="Publish the prepared Meta support source after Core", + detail=( + "After the matching Core annotated tag and exact main are published, use the shared " + "guarded Meta tag preview/local-tag/publish route. Commit/push reviewed preparation " + "and create a fresh durable run; do not update the current runtime binding." + ), + cwd=dashboard.meta_root, repo=unit.repo, status="needs-executor", + mutating=True, + ), + )) return tuple(steps) diff --git a/tools/release/govoplan_release/source_tag_batch.py b/tools/release/govoplan_release/source_tag_batch.py new file mode 100644 index 0000000..bc08723 --- /dev/null +++ b/tools/release/govoplan_release/source_tag_batch.py @@ -0,0 +1,792 @@ +"""Strict registered-source release contract for every source-tag batch. + +Meta is not a root Python package. Its nested developer package is released only +after a whole-batch source preflight and the matching immutable Core release. +No selected checkout supplies the developer-package generator or release +validation tooling. The trusted shared checker may load reviewed application +manifests; this is not an untrusted-code sandbox. +""" + +from __future__ import annotations + +from pathlib import Path +import hashlib +import os +import re +import stat + +from .git_state import collect_repository_snapshot, git, git_text +from .repository_tag import ( + _preview_repositories, + basic_blocker, + normalize_version, + ref_commit, + remote_tag_commit, + run, +) +from .source_provenance import registered_source_origin_issues +from .version_alignment import ( + repository_version_issues, + selected_release_webui_bundle_issues, + selected_webui_repository_names, +) +from .workspace import load_repository_specs, resolve_repo_path, resolve_workspace_root + +_OBJECT = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?\Z") + + +class SourceReceiptError(ValueError): + pass + + +def _owned_path(path, *, directory): + observed = path.lstat() + expected = stat.S_ISDIR if directory else stat.S_ISREG + if stat.S_ISLNK(observed.st_mode) or not expected(observed.st_mode): + raise SourceReceiptError( + "source authority must use real paths, without symlinks or special files" + ) + if observed.st_uid != os.geteuid(): + raise SourceReceiptError( + "source authority is not owned by the current operator" + ) + if observed.st_mode & 0o022: + raise SourceReceiptError("source authority is group/world writable") + return observed + + +def _trusted_ancestry(path): + # Same ownership/mode policy as the publisher's trust-path guard. A sticky + # shared ancestor such as /tmp may contain an owned, non-writable child; + # the workspace/repository themselves are never given that exception. + for ancestor in (path, *path.parents): + observed = ancestor.lstat() + if stat.S_ISLNK(observed.st_mode) or not stat.S_ISDIR(observed.st_mode): + raise SourceReceiptError( + "source ancestry must contain real directories, not symlinks" + ) + if observed.st_uid not in {0, os.geteuid()}: + raise SourceReceiptError("source ancestry has an untrusted owner") + if observed.st_mode & 0o022 and not observed.st_mode & stat.S_ISVTX: + raise SourceReceiptError("source ancestry is group/world writable") + + +def _git_pointer(path): + _owned_path(path, directory=False) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as source: + observed = os.fstat(source.fileno()) + if not stat.S_ISREG(observed.st_mode) or not 0 < observed.st_size <= 4096: + raise SourceReceiptError("Git metadata pointer is invalid or oversized") + value = source.read(observed.st_size + 1) + if len(value) != observed.st_size: + raise SourceReceiptError("Git metadata pointer changed during validation") + return value.decode("utf-8").strip() + + +def _git_tree(root): + pending = [(root, 0)] + count = 0 + while pending: + directory, depth = pending.pop() + _owned_path(directory, directory=True) + if depth > 128: + raise SourceReceiptError( + "Git metadata exceeds its trust-validation depth limit" + ) + with os.scandir(directory) as entries: + for entry in entries: + count += 1 + if count > 500_000: + raise SourceReceiptError( + "Git metadata exceeds its trust-validation entry limit" + ) + candidate = Path(entry.path) + observed = candidate.lstat() + if stat.S_ISDIR(observed.st_mode): + pending.append((candidate, depth + 1)) + else: + _owned_path(candidate, directory=False) + + +def _filesystem_identity(path, observed): + return [ + str(path), + observed.st_dev, + observed.st_ino, + observed.st_uid, + observed.st_gid, + stat.S_IMODE(observed.st_mode), + ] + + +def _source_filesystem(*, path, workspace): + """Validate source/Git ownership before invoking even read-only Git.""" + if path.absolute() != path.resolve() or not path.resolve().is_relative_to( + workspace.resolve() + ): + raise SourceReceiptError( + "source checkout leaves the private workspace or traverses a symlink" + ) + _trusted_ancestry(path) + workspace_info = _owned_path(workspace, directory=True) + repo_info = _owned_path(path, directory=True) + marker = path / ".git" + marker_info = marker.lstat() + if stat.S_ISDIR(marker_info.st_mode): + git_dir = marker + else: + value = _git_pointer(marker) + if not value.startswith("gitdir: "): + raise SourceReceiptError("Git worktree pointer is invalid") + git_dir = Path(os.path.abspath(path / value.removeprefix("gitdir: "))) + if git_dir.absolute() != git_dir.resolve() or not git_dir.resolve().is_relative_to( + workspace.resolve() + ): + raise SourceReceiptError( + "Git checkout metadata must stay inside the trusted workspace" + ) + _trusted_ancestry(git_dir) + _owned_path(git_dir, directory=True) + common_dir = git_dir + common_pointer = git_dir / "commondir" + if common_pointer.exists() or common_pointer.is_symlink(): + common_dir = Path(os.path.abspath(git_dir / _git_pointer(common_pointer))) + identities = { + "workspace": _filesystem_identity(workspace, workspace_info), + "checkout": _filesystem_identity(path, repo_info), + } + scanned = set() + for label, directory in ( + ("git_directory", git_dir), + ("git_common_directory", common_dir), + ): + if ( + directory.absolute() != directory.resolve() + or not directory.resolve().is_relative_to(workspace.resolve()) + ): + raise SourceReceiptError( + "Git checkout metadata must stay inside the trusted workspace" + ) + _trusted_ancestry(directory) + observed = _owned_path(directory, directory=True) + if observed.st_mode & 0o700 != 0o700: + raise SourceReceiptError( + "Git metadata target must be readable and writable by its operator" + ) + identities[label] = _filesystem_identity(directory, observed) + if directory not in scanned: + _git_tree(directory) + scanned.add(directory) + identities["git_marker"] = _filesystem_identity( + marker, _owned_path(marker, directory=stat.S_ISDIR(marker_info.st_mode)) + ) + for candidate in ( + common_dir / "objects/info/alternates", + common_dir / "objects/info/http-alternates", + common_dir / "info/grafts", + ): + if candidate.exists() or candidate.is_symlink(): + raise SourceReceiptError( + "Git object alternates and grafts are not permitted" + ) + return identities + + +def _tracked_worktree(path): + tracked = git(path, "ls-files", "-v", "-z", timeout=30) + if tracked.returncode or len(tracked.stdout) > 16 * 1024 * 1024: + raise SourceReceiptError( + "tracked release inputs exceed their trust-validation limit" + ) + checked = {path} + tracked_paths = set() + names = tracked.stdout.split("\0") + if len(names) > 100_001: + raise SourceReceiptError( + "tracked release inputs exceed their trust-validation count limit" + ) + for entry in filter(None, names): + # git status deliberately hides assume-unchanged and skip-worktree + # paths. Never validate mutable working metadata and then tag different + # committed bytes because an index flag suppressed the dirty evidence. + if not entry.startswith("H "): + raise SourceReceiptError( + "hidden, sparse or unmerged tracked index entries are not permitted" + ) + name = entry[2:] + tracked_paths.add(name) + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise SourceReceiptError("tracked release input has an unsafe path") + candidate = path / relative + for parent in candidate.parents: + if parent == path: + break + if parent not in checked: + _owned_path(parent, directory=True) + checked.add(parent) + _owned_path(candidate, directory=False) + # Version/composition checks inspect existing files, including ignored + # paths. Their declarations must come from the selected committed source, + # not ignored working bytes absent from the tag's tree. + metadata = [ + path / name + for name in ( + "pyproject.toml", + "package.json", + "package-lock.json", + "webui/package.json", + "webui/package.release.json", + "webui/package-lock.json", + "webui/package-lock.release.json", + ) + ] + if path.name == "govoplan": + metadata.extend( + path / name + for name in ( + "packages/govoplan-meta/pyproject.toml", + "requirements-release.txt", + ) + ) + metadata.extend((path / "src").glob("**/backend/manifest.py")) + metadata.extend((path / "src").glob("*/__init__.py")) + for candidate in metadata: + if (candidate.exists() or candidate.is_symlink()) and candidate.relative_to( + path + ).as_posix() not in tracked_paths: + raise SourceReceiptError( + "selected release version/composition metadata must be tracked in the frozen source" + ) + + +def _receipt(*, spec, workspace, version, filesystem): + path = resolve_repo_path(spec, workspace) + _tracked_worktree(path) + snapshot = collect_repository_snapshot( + spec, workspace_root=workspace, target_tag=f"v{version}", online=False + ) + blocker = basic_blocker(snapshot=snapshot, version=version) + if blocker: + raise SourceReceiptError(blocker) + if ( + not snapshot.exists + or not snapshot.is_git + or not snapshot.has_head + or snapshot.errors + or snapshot.safe_directory_required + or snapshot.dirty + or snapshot.branch != "main" + or snapshot.upstream != "origin/main" + or snapshot.behind + ): + raise SourceReceiptError( + "requires a clean registered main checkout tracking origin/main, not behind" + ) + common = git_text(path, "rev-parse", "--path-format=absolute", "--git-common-dir") + if ( + not common + or Path(common).absolute() != Path(common).resolve() + or not Path(common).resolve().is_relative_to(workspace.resolve()) + or git_text(path, "rev-parse", "--show-toplevel") != str(path.resolve()) + ): + raise SourceReceiptError( + "Git checkout metadata must stay inside the trusted workspace" + ) + head = git_text(path, "rev-parse", "--verify", "HEAD") + if not _OBJECT.fullmatch(head): + raise SourceReceiptError("source HEAD is not an exact commit") + live = run( + ("git", "ls-remote", "--exit-code", "--heads", "origin", "refs/heads/main"), + cwd=path, + ) + lines = live.stdout.strip().splitlines() + if live.returncode or len(lines) != 1: + raise SourceReceiptError("could not verify live origin/main") + remote_main, separator, ref = lines[0].partition("\t") + if not separator or ref != "refs/heads/main" or not _OBJECT.fullmatch(remote_main): + raise SourceReceiptError("live origin/main returned an invalid source receipt") + if git(path, "merge-base", "--is-ancestor", remote_main, head).returncode != 0: + raise SourceReceiptError( + "live origin/main is unavailable locally or diverges; fetch and review before retrying" + ) + tag = f"v{version}" + local_object = git_text(path, "rev-parse", "--verify", f"refs/tags/{tag}") or None + if local_object: + if git_text(path, "cat-file", "-t", f"refs/tags/{tag}") != "tag": + raise SourceReceiptError("local immutable tag must be annotated") + if ref_commit(path, f"refs/tags/{tag}") != head: + raise SourceReceiptError( + "local immutable tag points to another commit, not HEAD" + ) + remote_tag = remote_tag_commit(path, remote="origin", tag=tag) + if remote_tag.error: + raise SourceReceiptError("could not verify the remote release tag") + if remote_tag.tag_object: + if not remote_tag.annotated: + raise SourceReceiptError("remote immutable tag must be annotated") + if remote_tag.commit != head: + raise SourceReceiptError( + "remote immutable tag points to another commit, not HEAD" + ) + if local_object and remote_tag.tag_object != local_object: + raise SourceReceiptError( + "local and remote immutable tag annotation objects differ" + ) + return { + "head": head, + "branch": "main", + "upstream": "origin/main", + "origin": spec.remote, + "remote_main": remote_main, + "tag": tag, + "local_tag_object": local_object, + "remote_tag_object": remote_tag.tag_object, + "filesystem": filesystem, + } + + +def _collect_receipts(*, versions, specs, workspace): + filesystems = {} + for repo in versions: + if repo not in specs: + raise SourceReceiptError(f"{repo}: source repository is not registered") + filesystems[repo] = _source_filesystem( + path=resolve_repo_path(specs[repo], workspace), workspace=workspace + ) + issues = registered_source_origin_issues( + repo_versions=versions, workspace=workspace, remote="origin" + ) + if issues: + raise SourceReceiptError( + "; ".join(f"{issue.repo}: {issue.message}" for issue in issues) + ) + receipts = {} + for repo, version in versions.items(): + if repo not in specs: + raise SourceReceiptError(f"{repo}: source repository is not registered") + try: + receipts[repo] = _receipt( + spec=specs[repo], + workspace=workspace, + version=version, + filesystem=filesystems[repo], + ) + except SourceReceiptError as exc: + raise SourceReceiptError(f"{repo}: {exc}") from exc + return receipts + + +def _bundle_input_receipt(*, selected, workspace, push): + """Freeze only Core files used by the already-applicable WebUI gate. + + These are read-only composition inputs, not a new Core-tag/version or + clean-Core prerequisite. Local module candidates intentionally need none. + """ + if not push and "govoplan-core" not in selected: + return {} + if not selected_webui_repository_names( + repo_versions=dict.fromkeys(selected, ""), workspace=workspace + ): + return {} + result = {} + core = workspace / "govoplan-core" + for relative in ("webui/package.release.json", "webui/package-lock.release.json"): + path = core / relative + if not path.exists() and not path.is_symlink(): + result[relative] = None # The unchanged shared gate explains missing input. + continue + _trusted_ancestry(path.parent) + _owned_path(core, directory=True) + _owned_path(path.parent, directory=True) + observed = _owned_path(path, directory=False) + if not 0 < observed.st_size <= 16 * 1024 * 1024: + raise SourceReceiptError( + "Core release-bundle input exceeds its 16 MiB limit" + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as source: + before = os.fstat(source.fileno()) + if ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) != ( + observed.st_dev, + observed.st_ino, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ): + raise SourceReceiptError( + "Core release-bundle input changed during inspection" + ) + content = source.read(before.st_size + 1) + after = os.fstat(source.fileno()) + if len(content) != before.st_size or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ): + raise SourceReceiptError( + "Core release-bundle input changed during inspection" + ) + result[relative] = { + "file": _filesystem_identity(path, observed), + "sha256": hashlib.sha256(content).hexdigest(), + } + return result + + +def _frozen_receipts( + *, expected, versions, specs, workspace, selected, push, bundle_inputs +): + actual = _collect_receipts(versions=versions, specs=specs, workspace=workspace) + for repo in expected: + if actual[repo] != expected[repo]: + raise SourceReceiptError( + f"{repo}: source receipt changed after whole-batch preflight" + ) + for repo in versions: + issues = repository_version_issues( + resolve_repo_path(specs[repo], workspace), expected_version=versions[repo] + ) + if issues: + raise SourceReceiptError( + f"{repo}: version/composition changed after whole-batch preflight" + ) + if ( + _bundle_input_receipt(selected=selected, workspace=workspace, push=push) + != bundle_inputs + ): + raise SourceReceiptError( + "Core release-bundle input receipt changed after whole-batch preflight" + ) + if push or "govoplan-core" in selected: + if selected_release_webui_bundle_issues( + repo_versions={repo: versions[repo] for repo in selected}, + workspace=workspace, + ): + raise SourceReceiptError( + "release WebUI composition changed after whole-batch preflight" + ) + + +def _require_core(receipts, *, push): + core = receipts["govoplan-core"] + if not core["local_tag_object"] or ( + push and (not core["remote_tag_object"] or core["remote_main"] != core["head"]) + ): + raise SourceReceiptError( + "Meta requires the matching annotated Core tag locally and, for publication, remotely" + ) + + +def _blocked(*, selected, apply, push, detail, rows=()): # noqa: A002 + known = {row["repo"]: row for row in rows} + identified = next( + (repo for repo in selected if detail.startswith(repo + ":")), None + ) + return { + "status": "blocked", + "apply": apply, + "push": push, + "remote": "origin", + "detail": "whole-batch source preflight failed; no selected repository was mutated", + "repositories": [ + { + **known.get(repo, {"repo": repo}), + "status": "blocked" + if ( + known.get(repo, {}).get("status") == "blocked" + or (not rows and (identified is None or repo == identified)) + ) + else "skipped", + "detail": known[repo]["detail"] + if known.get(repo, {}).get("status") == "blocked" + else detail, + } + for repo in selected + ], + } + + +def tag_source_batch( + *, repos, repo_versions, workspace_root, remote, message, apply, push +): # noqa: A002 + workspace = resolve_workspace_root(workspace_root) + has_meta = "govoplan" in repos + selected = tuple(dict.fromkeys(repos)) + if has_meta: + selected = tuple(repo for repo in selected if repo != "govoplan") + ( + "govoplan", + ) + if not selected: + return { + "status": "noop", + "apply": apply, + "push": push, + "remote": remote.strip() or "origin", + "repositories": [], + } + versions = {repo: normalize_version(repo_versions.get(repo)) for repo in selected} + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + meta_version = versions.get("govoplan") + try: + if remote.strip() not in {"", "origin"}: + raise SourceReceiptError( + "Source-tag batches require the registered origin remote" + ) + if any(not version for version in versions.values()): + raise SourceReceiptError( + "every selected repository requires an explicit valid version" + ) + if has_meta and versions.get("govoplan-core", meta_version) != meta_version: + raise SourceReceiptError( + "Meta developer-package version must match the selected Core release" + ) + # Only Meta requires a frozen version-matched Core source/tag dependency. + if has_meta: + versions.setdefault("govoplan-core", meta_version) + receipts = _collect_receipts( + versions=versions, specs=specs, workspace=workspace + ) + if has_meta and "govoplan-core" not in selected: + _require_core(receipts, push=push) + core_issues = ( + repository_version_issues( + resolve_repo_path(specs["govoplan-core"], workspace), + expected_version=meta_version, + ) + if has_meta + else () + ) + if core_issues: + raise SourceReceiptError( + "Core source metadata must match the selected Meta version" + ) + bundle_inputs = _bundle_input_receipt( + selected=selected, workspace=workspace, push=push + ) + except (SourceReceiptError, OSError, ValueError) as exc: + return _blocked(selected=selected, apply=apply, push=push, detail=str(exc)) + + # Preserve the shared complete manifest, package/lock and immutable tag + # preflight. This invocation is always read-only; strict effects stay below. + try: + preview = _preview_repositories( + repos=selected, + repo_versions=repo_versions, + workspace_root=workspace, + remote="origin", + message=message, + push=push, + ) + except (OSError, ValueError) as exc: + return _blocked( + selected=selected, + apply=apply, + push=push, + detail=f"shared release preflight could not validate its inputs ({type(exc).__name__})", + ) + rows = preview["repositories"] + if preview["status"] in {"blocked", "partial"}: + if not apply: + return preview + return _blocked( + selected=selected, + apply=True, + push=push, + detail="shared release preflight failed", + rows=rows, + ) + if not apply: + rows = [ + { + **row, + "status": "planned", + "detail": "existing annotated release tag requires atomic main publication", + } + if push + and receipts[row["repo"]]["remote_main"] != receipts[row["repo"]]["head"] + else row + for row in rows + ] + preview = {**preview, "repositories": rows} + if any(row["status"] == "planned" for row in rows): + preview["status"] = "planned" + return { + **preview, + "source_receipts": receipts, + "source_contract": "registered-meta-batch-v1" + if has_meta + else "registered-source-batch-v1", + "bundle_input_receipts": bundle_inputs, + } + + results = [] + effected = False + for row in rows: + repo = row["repo"] + receipt = receipts[repo] + path = resolve_repo_path(specs[repo], workspace) + tag = receipt["tag"] + head = receipt["head"] + try: + # Recheck the entire frozen batch, including Meta and Core, before + # every effect. Earlier successful effects update only their exact + # anticipated tag/branch receipt fields below. + _frozen_receipts( + expected=receipts, + versions=versions, + specs=specs, + workspace=workspace, + selected=selected, + push=push, + bundle_inputs=bundle_inputs, + ) + if repo == "govoplan": + _require_core(receipts, push=push) + local_object = receipt["local_tag_object"] + created = False + if not local_object: + if receipt["remote_tag_object"]: + command = ( + "git", + "fetch", + "--no-tags", + "origin", + f"refs/tags/{tag}:refs/tags/{tag}", + ) + else: + command = ("git", "tag", "-a", tag, head, "-m", row["message"]) + created = True + effected = True + if run(command, cwd=path).returncode: + raise SourceReceiptError( + "annotated local release tag could not be created or retrieved" + ) + local_object = git_text( + path, "rev-parse", "--verify", f"refs/tags/{tag}" + ) + if ( + not local_object + or ref_commit(path, f"refs/tags/{tag}") != head + or git_text(path, "cat-file", "-t", f"refs/tags/{tag}") != "tag" + or ( + receipt["remote_tag_object"] + and local_object != receipt["remote_tag_object"] + ) + ): + raise SourceReceiptError("local release tag postcondition failed") + receipt["local_tag_object"] = local_object + _frozen_receipts( + expected=receipts, + versions=versions, + specs=specs, + workspace=workspace, + selected=selected, + push=push, + bundle_inputs=bundle_inputs, + ) + if push and ( + receipt["remote_tag_object"] != local_object + or receipt["remote_main"] != head + ): + # Pin both effects to verified objects, not mutable HEAD/tag + # names. No force, retagging, fallback or non-atomic retry. + command = ( + "git", + "push", + "--atomic", + "origin", + f"{head}:refs/heads/main", + f"{local_object}:refs/tags/{tag}", + ) + effected = True + if run(command, cwd=path).returncode: + raise SourceReceiptError( + "atomic main and annotated tag publication failed; inspect receipts before retrying" + ) + receipt["remote_main"] = head + receipt["remote_tag_object"] = local_object + # Verify remote main AND exact annotated object, as well as local + # source state. A successful Git exit alone is never a receipt. + _frozen_receipts( + expected=receipts, + versions=versions, + specs=specs, + workspace=workspace, + selected=selected, + push=push, + bundle_inputs=bundle_inputs, + ) + results.append( + { + **row, + "status": "published" if push else "tagged" if created else "noop", + "detail": "verified strict registered-source release" + if not receipt["remote_tag_object"] or created + else "verified strict registered-source release; immutable annotation already published or present", + "after_local_tag_commit": head, + "after_local_tag_object": local_object, + "after_remote_tag_commit": head + if receipt["remote_tag_object"] + else None, + "after_remote_tag_object": receipt["remote_tag_object"], + "after_remote_main_commit": receipt["remote_main"], + } + ) + except (SourceReceiptError, OSError, ValueError) as exc: + results.append( + { + **row, + "status": "failed" if effected else "blocked", + "detail": str(exc), + } + ) + results.extend( + { + **later, + "status": "skipped", + "detail": "earlier strict source effect or receipt failed", + } + for later in rows[len(results) :] + ) + return { + "status": "partial" if effected else "blocked", + "apply": True, + "push": push, + "remote": "origin", + "repositories": results, + } + status = ( + "published" + if push + else "tagged" + if any(row["status"] == "tagged" for row in results) + else "noop" + ) + return { + "status": status, + "apply": True, + "push": push, + "remote": "origin", + "repositories": results, + "source_receipts": receipts, + "source_contract": "registered-meta-batch-v1" + if has_meta + else "registered-source-batch-v1", + "bundle_input_receipts": bundle_inputs, + } diff --git a/tools/release/govoplan_release/version_alignment.py b/tools/release/govoplan_release/version_alignment.py index f314db3..d86753e 100644 --- a/tools/release/govoplan_release/version_alignment.py +++ b/tools/release/govoplan_release/version_alignment.py @@ -6,12 +6,13 @@ from dataclasses import dataclass import json from pathlib import Path import re +import runpy import subprocess import tomllib -from .git_state import collect_versions, sanitized_git_environment +from .git_state import collect_versions, registered_developer_meta_path, sanitized_git_environment from .registry_reference import registry_artifact_conflicts, registry_entry_source -from .workspace import load_repository_specs, resolve_repo_path +from .workspace import META_ROOT, load_repository_specs, resolve_repo_path _PYTHON_RELEASE_REF = re.compile( @@ -43,6 +44,7 @@ def repository_version_issues( versions = collect_versions(repo_path) declared = { "pyproject.toml": versions.pyproject, + "packages/govoplan-meta/pyproject.toml": versions.developer_meta, "package.json": versions.package, "webui/package.json": versions.webui_package, "webui/package.release.json": _json_version(repo_path / "webui" / "package.release.json") @@ -83,6 +85,8 @@ def repository_version_issues( for source, version in declared.items() if version is not None and version != canonical_version ] + if registered_developer_meta_path(repo_path) is not None: + issues.extend(developer_meta_composition_issues(repo_path)) if expected_version is not None and canonical_version.removeprefix("v") != expected_version.removeprefix("v"): issues.append( @@ -120,6 +124,34 @@ def repository_version_issues( return tuple(issues) +def developer_meta_composition_issues(repo_path: Path) -> tuple[VersionAlignmentIssue, ...]: + """Compare the real nested package with the trusted generator's exact output. + + Never execute a generator from a selected checkout. Only the installed + operator tooling provides code; selected TOML/requirements are data inputs. + """ + package_path = registered_developer_meta_path(repo_path) + if package_path is None: + return (VersionAlignmentIssue(repo_path.name, "repository", "registered Meta support repository", "", "nested developer-package identity is not registered"),) + source = "packages/govoplan-meta/pyproject.toml" + try: + current = package_path.read_text(encoding="utf-8") + project = tomllib.loads(current).get("project") + if not isinstance(project, dict) or project.get("name") != "govoplan": + return (VersionAlignmentIssue("govoplan", source + ":project.name", "govoplan", str(project.get("name") if isinstance(project, dict) else ""), "developer meta-package identity must be exact"),) + # META_ROOT belongs to the running operator tools, not repo_path. + generator = runpy.run_path(str(META_ROOT / "tools/release/generate-developer-meta-package.py")) + expected = generator["render"]( + workspace=repo_path.parent, + requirements=repo_path / "requirements-release.txt", + ) + except (OSError, UnicodeError, KeyError, ValueError, TypeError) as exc: + return (VersionAlignmentIssue("govoplan", source, "readable exact developer composition and Core version", type(exc).__name__, "developer meta-package composition could not be validated"),) + if current != expected: + return (VersionAlignmentIssue("govoplan", source, "trusted generator output matching Core and release requirements", "stale composition", "developer meta-package must exactly match the generator --check contract"),) + return () + + def selected_repository_version_issues( *, repo_versions: dict[str, str], @@ -165,6 +197,11 @@ def selected_repository_version_issues( return tuple(issues) +def selected_webui_repository_names(*, repo_versions: dict[str, str], workspace: Path) -> tuple[str, ...]: + """Identify the exact selections for which Core's WebUI inputs are relevant.""" + return tuple(repo for repo in sorted(repo_versions) if repo != "govoplan-core" and (workspace / repo / "webui/package.json").exists()) + + def selected_release_webui_bundle_issues( *, repo_versions: dict[str, str], @@ -177,6 +214,8 @@ def selected_release_webui_bundle_issues( immutable release package input and lockfile will actually install. """ + if not selected_webui_repository_names(repo_versions=repo_versions, workspace=workspace): + return () core_webui = workspace / "govoplan-core" / "webui" release_package_path = core_webui / "package.release.json" release_lock_path = core_webui / "package-lock.release.json" diff --git a/tools/release/govoplan_release/version_metadata.py b/tools/release/govoplan_release/version_metadata.py index d95a425..16e8a43 100644 --- a/tools/release/govoplan_release/version_metadata.py +++ b/tools/release/govoplan_release/version_metadata.py @@ -34,6 +34,19 @@ def version_metadata_mutations( ) -> tuple[VersionFileMutation, ...]: """Render all recognized repository version files without writing them.""" + from .git_state import registered_developer_meta_path + + if registered_developer_meta_path(repo_path) is not None: + from .meta_preparation import MetaPreparationError, PACKAGE, preview_meta_mutation + + try: + _receipt, before, after = preview_meta_mutation( + repo_path=repo_path, target_version=target_version, + ) + except (MetaPreparationError, OSError, ValueError, KeyError, TypeError) as exc: + raise VersionMetadataError(str(exc)) from exc + return (VersionFileMutation(PACKAGE, before, after),) if before != after else () + version = target_version.removeprefix("v") candidates: list[tuple[Path, str]] = [] if (repo_path / "pyproject.toml").is_file(): @@ -115,6 +128,14 @@ def apply_version_metadata_mutations( ) -> tuple[str, ...]: """Apply one deterministic version update, rolling back on write failure.""" + from .git_state import registered_developer_meta_path + + if registered_developer_meta_path(repo_path) is not None: + raise VersionMetadataError( + "Meta is prepared outside durable runs with prepare-developer-meta-package.py; " + "review its complete generated composition, commit, and create a fresh run." + ) + mutations = version_metadata_mutations( repo_path, target_version=target_version, diff --git a/tools/release/prepare-developer-meta-package.py b/tools/release/prepare-developer-meta-package.py new file mode 100644 index 0000000..63a64a0 --- /dev/null +++ b/tools/release/prepare-developer-meta-package.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Preview or explicitly prepare Meta outside a durable release run.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from govoplan_release.meta_preparation import ( + MetaPreparationAmbiguous, + MetaPreparationError, + _read_input, + prepare_developer_meta_package, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--target-version", required=True) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--receipt", type=Path) + parser.add_argument("--confirm-out-of-run", action="store_true") + args = parser.parse_args() + try: + expected = None + if args.apply: + if args.receipt is None: + raise MetaPreparationError( + "Apply requires the reviewed preview JSON via --receipt." + ) + payload, _ = _read_input(args.receipt) + expected = json.loads(payload)["receipt"] + result = prepare_developer_meta_package( + repo_path=args.workspace.absolute() / "govoplan", + target_version=args.target_version, + apply=args.apply, + expected_receipt=expected, + confirm_out_of_run=args.confirm_out_of_run, + ) + except (MetaPreparationError, OSError, ValueError, KeyError, TypeError) as exc: + status = "needs-reconciliation" if isinstance(exc, MetaPreparationAmbiguous) else "blocked" + print(json.dumps({"status": status, "detail": str(exc)})) + return 1 + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html index e7d7838..7ffef39 100644 --- a/tools/release/webui/index.html +++ b/tools/release/webui/index.html @@ -2141,6 +2141,7 @@ function primaryVersion(repo) { const versions = repo.versions || {}; if (versions.pyproject) return versions.pyproject; + if (versions.developer_meta) return versions.developer_meta; if (versions.package) return versions.package; if (versions.webui_package) return versions.webui_package; if (Array.isArray(versions.manifests) && versions.manifests.length) return versions.manifests[0];