diff --git a/.gitea/workflows/dependency-audit.yml b/.gitea/workflows/dependency-audit.yml index 5cdfecc..c2ccbe5 100644 --- a/.gitea/workflows/dependency-audit.yml +++ b/.gitea/workflows/dependency-audit.yml @@ -42,8 +42,7 @@ jobs: working-directory: govoplan run: | python -m venv .venv - .venv/bin/python -m pip install --upgrade pip - .venv/bin/python -m pip install -r requirements-release.txt + .venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-release.txt --python .venv/bin/python --upgrade-pip .venv/bin/python -m pip install 'pip-audit>=2.9,<3' - name: Install WebUI release dependencies working-directory: govoplan diff --git a/.gitea/workflows/module-matrix.yml b/.gitea/workflows/module-matrix.yml index 29c33c5..a8b6e82 100644 --- a/.gitea/workflows/module-matrix.yml +++ b/.gitea/workflows/module-matrix.yml @@ -38,8 +38,7 @@ jobs: working-directory: govoplan run: | python -m venv .venv - .venv/bin/python -m pip install --upgrade pip - .venv/bin/python -m pip install -r requirements-release.txt + .venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-release.txt --python .venv/bin/python --upgrade-pip .venv/bin/python -m pip install '../govoplan-core[dev]' - name: Install WebUI release dependencies with test scripts working-directory: govoplan diff --git a/.gitea/workflows/release-integration.yml b/.gitea/workflows/release-integration.yml index a79e55b..1e33c60 100644 --- a/.gitea/workflows/release-integration.yml +++ b/.gitea/workflows/release-integration.yml @@ -37,8 +37,7 @@ jobs: working-directory: govoplan run: | python -m venv .venv - .venv/bin/python -m pip install --upgrade pip - .venv/bin/python -m pip install -r requirements-release.txt + .venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-release.txt --python .venv/bin/python --upgrade-pip .venv/bin/python -m pip install '../govoplan-core[dev]' - name: Install WebUI release dependencies working-directory: govoplan diff --git a/.gitleaks.toml b/.gitleaks.toml index 310db35..f6058cc 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -3,7 +3,6 @@ title = "GovOPlaN secret scanning" [allowlist] description = "Repository examples and documented development placeholders" regexes = [ - '''dev-multimailer-api-key''', '''example(\.invalid|\.org|\.com)''', '''postgresql(\+psycopg)?://[^:@/\s]+:\*\*\*@''', '''''', diff --git a/README.md b/README.md index d4f0e9c..7258947 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,7 @@ Create the whole-product development virtualenv in this meta repository: ```sh python3 -m venv .venv -./.venv/bin/python -m pip install --upgrade pip -./.venv/bin/python -m pip install -r requirements-dev.txt +./.venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-dev.txt --python ./.venv/bin/python --upgrade-pip ``` The meta venv is the default Python environment for launch, check, release, and @@ -55,6 +54,24 @@ Update generated repository type notes in all READMEs: ./tools/repo/update-repository-type-notes.py ``` +Regenerate the human-readable repository link index: + +```sh +./tools/repo/generate-repository-index.py +``` + +Synchronize the Python environment after package metadata changes: + +```sh +./.venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-dev.txt --python ./.venv/bin/python +``` + +Run the static cross-repository module contract check: + +```sh +./tools/checks/check-contracts.sh +``` + Run the consolidated focused verification suite: ```sh @@ -76,6 +93,12 @@ Run installer rollback drills: Release, catalog, Gitea, security-audit, and cross-repository maintenance commands should also be called from this repository through `tools/`. +Start the local release console: + +```sh +./.venv/bin/python tools/release/release-console.py +``` + ## Configuration The repository root `.env.example` is the self-hosted operator template for a @@ -89,7 +112,13 @@ such as `~/.config/gitea/gitea.env` and be passed with `--env-file`. The repository categories are documented in `docs/REPOSITORY_STRUCTURE.md`. The machine-readable list lives in -`repositories.json`. +`repositories.json`; the clickable human-readable index is +`docs/REPOSITORY_INDEX.md`. + +Meta ownership and module install/contract boundaries are documented in +`docs/META_REPO_SCAN.md` and `docs/MODULE_CONTRACTS_AND_INSTALLS.md`. +Frontend layout principles for module pages are documented in +`docs/FRONTEND_LAYOUT_PRINCIPLES.md`. # GovOPlaN Docker diff --git a/docs/FRONTEND_LAYOUT_PRINCIPLES.md b/docs/FRONTEND_LAYOUT_PRINCIPLES.md new file mode 100644 index 0000000..b84b9a0 --- /dev/null +++ b/docs/FRONTEND_LAYOUT_PRINCIPLES.md @@ -0,0 +1,57 @@ +# GovOPlaN Frontend Layout Principles + +GovOPlaN modules should choose their page layout by the kind of work the user is +doing, not by the repository that owns the feature. + +## Structured Data Directories + +Use a full-available-space workspace for structured data directories: files, +addresses, calendars, records, mailboxes, document stores, and similar domains +where the primary task is browsing, selecting, filtering, inspecting, and acting +on related objects. + +Principles: + +- The module route should use the full available content area. +- Do not add a separate page heading row above the main workspace. +- Prefer persistent navigation panes, such as tree panels, source panels, folder + panels, calendar list panels, or mailbox folder panels. +- Keep collection navigation and collection-level actions close to the relevant + pane header. +- Use bounded widths for navigation/list panes and let the main detail/content + pane take the remaining space. +- Keep filtering controls inside the pane they affect. +- Use overlays, toasts, or floating alerts for transient messages so the + workspace height does not change. + +This pattern is appropriate when the user is working inside one coherent data +domain and needs spatial continuity. + +## Workflow And Configuration Surfaces + +Use the standard heading/menu/card visual language for workflow structures, +settings, administration, dashboards, and pages that collect essentially +unrelated areas. + +Principles: + +- A page heading and subnavigation are appropriate when the page explains a + task, workflow stage, or administrative area. +- Cards are appropriate for repeated independent panels, settings groups, + summaries, and dashboard widgets. +- Collapsible panels and segmented controls are appropriate when a dense + configuration area needs controlled disclosure. +- Avoid forcing workflow/configuration pages into a file-explorer style unless + the primary interaction is genuinely directory browsing. + +This pattern is appropriate when the user is comparing or configuring separate +concerns rather than navigating one structured object space. + +## Shared Components + +Reusable layout components belong in `govoplan-core` WebUI. Modules may consume +shared components from core, but must not import another module's private UI +components directly. + +When a module-specific component becomes generally useful, promote it to core +with a parameterized API before reusing it elsewhere. diff --git a/docs/META_REPO_SCAN.md b/docs/META_REPO_SCAN.md new file mode 100644 index 0000000..bcf2f0d --- /dev/null +++ b/docs/META_REPO_SCAN.md @@ -0,0 +1,83 @@ +# Meta Repository Scan + +Scan date: 2026-07-13. + +This scan checked local repositories under `/mnt/DATA/git` listed in +`repositories.json`. + +## Repository Inventory + +`repositories.json` already lists every checked-out GovOPlaN repository. + +Checked-out repositories not listed in `repositories.json`: none. + +Repositories listed in `repositories.json` but not checked out locally: none. + +The human-readable link index is `docs/REPOSITORY_INDEX.md`; the JSON file +remains the machine-readable source of truth. + +## Meta-Owned Content + +These items are correctly owned by the meta repository: + +- `.gitea/workflows`: cross-repository CI and audit workflows. +- `.gitleaks.toml`: whole-workspace secret scan policy. +- `audit-reports`: whole-workspace audit output. +- `dev/postgres`: shared development PostgreSQL service. +- `dev/production-like`: production-like product validation composition. +- `tools/checks`: whole-product checks and audit launchers. +- `tools/gitea`: issue/wiki/label/backlog tooling. +- `tools/launch`: product launch entry points. +- `tools/release`: release catalog, lock, push, and release console tooling. +- `tools/repo`: repository bootstrap, status, and metadata tooling. + +## Module-Local Content That Should Stay Local + +These paths look operational, but they are tied to one module's protocol or +transport behavior and should stay in the owning repository for now: + +- `govoplan-campaign/dev/mail-testbed`: mail transport testbed for campaign. +- `govoplan-files/dev/connectors`: WebDAV, Nextcloud, and SMB connector smoke + environment for files. + +The meta repo can later wrap these with aggregate commands, but the test-bed +definitions should remain close to the module code unless they become a shared +product deployment profile. + +## Generated Or Local-Only Content + +These should not move to the meta repo; they should be ignored or cleaned: + +- `__pycache__` directories under `tools/gitea` and module test beds. +- `.venv` and `.ruff_cache` directories. +- Populated `.env` files such as `govoplan-campaign/dev/mail-testbed/.env`. + +## Repo-Local Workflow Files + +Most module repositories have `.gitea/ISSUE_TEMPLATE` installed. These are +repo-local generated copies of the shared workflow templates and are fine to +keep in each repository. + +Repositories currently missing a `.gitea` directory in the local checkout: + +- `govoplan-dashboard` +- `govoplan-evaluation` +- `govoplan-poll` +- `govoplan-rest` +- `govoplan-soap` + +If those repositories should use the shared issue templates, run the meta repo +installer from `govoplan/tools/gitea/gitea-install-workflow.py`. + +## Do Not Move + +The following stay in each module repository: + +- `pyproject.toml` and package dependency metadata. +- Backend manifests and migrations. +- Module-owned tests. +- Module READMEs and module-specific docs. +- WebUI package manifests and module frontend source. + +Those files describe or implement the module itself. The meta repo should +catalog and orchestrate them, not become the owner of module code. diff --git a/docs/MODULE_CONTRACTS_AND_INSTALLS.md b/docs/MODULE_CONTRACTS_AND_INSTALLS.md new file mode 100644 index 0000000..2fc9d65 --- /dev/null +++ b/docs/MODULE_CONTRACTS_AND_INSTALLS.md @@ -0,0 +1,120 @@ +# Module Contracts and Install Boundaries + +## Why Some Changes Require `pip install` + +GovOPlaN discovers runtime modules through Python package entry points in the +`govoplan.modules` group. Core reads those entry points with +`importlib.metadata.entry_points()`, imports the configured manifest factory, and +then builds the module registry from the returned `ModuleManifest`. + +That means the Python environment must know that a package exists before core can +discover it. + +In development, `requirements-dev.txt` installs modules with `-e +../govoplan-module`. With editable installs: + +- normal Python source changes are picked up after the process reloads; +- manifest code changes are picked up after the process reloads; +- new imports inside an already installed package are picked up after reload. + +`pip install` is still needed when package metadata changes: + +- a repository was not installed in the environment before; +- a `pyproject.toml` entry point is added, renamed, or removed; +- package dependencies change; +- optional extras change; +- package names or import roots change; +- console scripts or other installed metadata change; +- release installs need a different tag, wheel, or source ref. + +The same principle applies to WebUI packages: source changes are local during +development, but `package.json` dependency or export changes require an install +step so the consuming app sees the correct package metadata. + +## Current Contract Mechanism + +Modules already announce contracts through `ModuleManifest`: + +- `dependencies` +- `optional_dependencies` +- `required_capabilities` +- `optional_capabilities` +- `provides_interfaces` +- `requires_interfaces` +- `capability_factories` +- permissions and role templates +- route factories, migrations, docs, lifecycle hooks, and frontend metadata + +Core validates the active manifest graph when it builds the registry. Release +tooling can inspect those manifests to calculate compatibility and migration +impact. + +## What Core Can and Cannot Pick Up Automatically + +Core can pick up contract changes automatically after reload when the changed +module package is already installed and importable. + +Core cannot discover a new module or new entry point that has not been installed +into the environment, because there is no distribution metadata to enumerate. + +Core also cannot notify every module about a contract change at edit time by +itself. The runtime registry is built from installed/importable packages. The +right place for cross-module announcement is the meta repo tooling and CI: + +- scan manifests across all repositories; +- build an impact graph from provided/required interfaces and capabilities; +- run affected module tests; +- post Gitea issue/release notes for affected modules; +- block releases when a required interface is missing or incompatible. + +## Mitigation Strategy + +Use three layers: + +1. Editable development environment. + Keep `requirements-dev.txt` in the meta repo as the one workspace installer. + Source edits then need process reloads, not repeated full installs. + +2. Versioned runtime contracts. + Keep adding and tightening `provides_interfaces`, `requires_interfaces`, and + capability protocols. Treat interface names and versions as public module + contracts. + +3. Meta-level contract audit. + The meta repo statically reads `src/**/backend/manifest.py` files across all + repositories and validates provided/required interface ranges without + importing the packages. CI blocks missing or incompatible required + interfaces before release installs are attempted. + +Run the static graph check with: + +```sh +./tools/checks/check-contracts.sh +``` + +Use `--json` when the release console or another automation needs structured +provider/consumer impact data. + +## Practical Rule + +Do not run `pip install` for every code edit. Run it when package metadata or the +set of installed packages changes. + +For normal development: + +```sh +./.venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-dev.txt --python ./.venv/bin/python +``` + +Then restart the server when Python code or manifests change. The development +launcher runs this helper automatically by default; set +`GOVOPLAN_AUTO_SYNC_PYTHON=0` to disable that preflight. + +For release validation: + +```sh +./.venv/bin/python tools/repo/sync-python-environment.py --requirements requirements-release.txt --python ./.venv/bin/python +``` + +That install is necessary because the release environment intentionally resolves +tagged package refs, not local editable source trees. diff --git a/docs/RELEASE_CONSOLE.md b/docs/RELEASE_CONSOLE.md new file mode 100644 index 0000000..2c11d36 --- /dev/null +++ b/docs/RELEASE_CONSOLE.md @@ -0,0 +1,194 @@ +# GovOPlaN Release Console + +The release console is a local operator tool for planning and executing +GovOPlaN releases. It belongs to the `govoplan` meta repository because it works +across all local checkouts, release scripts, module manifests, migration audits, +catalog files, Git state, and signing keys. + +The current implementation has a read-only dashboard plus guarded local +candidate/publish actions: + +- inspect repositories from `repositories.json` +- show dirty, ahead, behind, missing, no-HEAD, and tag state +- show local package catalog and keyring state +- optionally run release/dev migration audits +- propose next actions without executing them +- compare local catalog/keyring JSON with the published channel and public + keyring when online checks are enabled +- configure target versions per release unit in the web UI +- select the repositories that should advance through checkboxes +- generate dry-run selective release plans for independently versioned packages +- generate signed catalog candidates for the selected release units +- preview/apply/push reviewed catalog candidates behind explicit confirmations + +Start it from the meta repository: + +```sh +./.venv/bin/python tools/release/release-console.py +``` + +The server binds to `127.0.0.1` by default and prints a URL containing a local +API token. Open that URL in a browser on the same machine. + +The web UI starts with a repository table. Each repository can be checked +independently and assigned its own target version. Repositories without version +metadata remain visible so they can be planned as initial releases. `Build Plan` +shows the dry-run commands for the selected rows, and `Generate Candidate` +creates a signed catalog candidate that advances only selected repositories that +already have a catalog entry. + +The release-control area above the repository table is read-only and is meant +to become the central release cockpit. It shows: + +- local and published channel health +- catalog/keyring drift +- signature and trusted-key status +- published module versions and refs +- local checkout version drift against the catalog +- catalog-declared interface compatibility + +The target-version control can generate the next major, minor, or subversion +from the current base version. Manual target input accepts explicit versions +such as `0.2.0` or `0.2.0-alpha1`, but requires the first three version numbers +to move forward. + +Plain repository pushes are separate from catalog publication. `Preview Push` +shows the selected repository push commands. `Push Selected` requires `PUSH` in +the repository push confirmation field. + +The catalog workflow panel can also operate on the same selected rows: + +- `Generate` creates a signed candidate below `runtime/release-candidates/`. +- `Preview` validates a candidate and shows what would be copied into the + website repository. +- `Apply + Tag` requires `APPLY` in the confirmation field. +- `Push` requires `PUSH` in the confirmation field. + +The default signing key is +`$HOME/.config/govoplan/release-keys/release-key-1.pem` when no signing key is +entered in the UI. + +Generate a selective release plan from the terminal: + +```sh +./.venv/bin/python tools/release/release-plan.py \ + --repo govoplan-files \ + --target-version 0.1.9 \ + --channel stable \ + --online +``` + +`--online` compares the local catalog/keyring with the published channel and +published keyring. Use `--remote-tags` only when the plan also needs to check +Git remotes for tag existence; that can be slower across the full repository +set. + +Build a signed selective catalog candidate: + +```sh +KEY_DIR="$HOME/.config/govoplan/release-keys" + +./.venv/bin/python tools/release/release-catalog.py selective \ + --repo-version govoplan-files=0.1.9 \ + --channel stable \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" +``` + +This writes candidate catalog/keyring files below `runtime/release-candidates/`, +generates the browsable module directory below `modules/`, validates the signed +candidate with the module installer validator, and reports whether the candidate +catalog/keyring match the currently published channel. It does not publish to +the website repository. + +Regenerate the browsable module directory from an existing catalog/keyring: + +```sh +./.venv/bin/python tools/release/release-catalog.py module-directory \ + --catalog ../addideas-govoplan-website/public/catalogs/v1/channels/stable.json \ + --keyring ../addideas-govoplan-website/public/catalogs/v1/keyring.json \ + --output-dir runtime/module-directory-preview \ + --channel stable +``` + +Preview publication of a reviewed candidate: + +```sh +./.venv/bin/python tools/release/release-catalog.py publish-candidate \ + --candidate-dir runtime/release-candidates/stable-YYYYMMDD-HHMMSS \ + --channel stable +``` + +Apply the reviewed candidate into the website repository without pushing: + +```sh +./.venv/bin/python tools/release/release-catalog.py publish-candidate \ + --candidate-dir runtime/release-candidates/stable-YYYYMMDD-HHMMSS \ + --channel stable \ + --apply \ + --commit \ + --tag +``` + +Push is a separate explicit flag: + +```sh +./.venv/bin/python tools/release/release-catalog.py publish-candidate \ + --candidate-dir runtime/release-candidates/stable-YYYYMMDD-HHMMSS \ + --channel stable \ + --apply \ + --commit \ + --tag \ + --push +``` + +Published channels are expected below the public catalog base URL: + +- `https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json` +- `https://govoplan.add-ideas.de/catalogs/v1/keyring.json` + +The console treats channels as release artifacts. A package can advance without +forcing every repository to the same tag, but channel publication must preserve +the unchanged package versions, validate interface compatibility, sign the +updated catalog, and keep the published keyring healthy. + +## Published Module Directory + +The target release repository is an online, browsable module directory. The +catalog remains the machine-readable channel entry point, but the published +space should also expose a tree that operators and installations can inspect: + +- `/catalogs/v1/channels/.json` describes the active channel state. +- `/catalogs/v1/keyring.json` publishes trusted release signing keys. +- `/catalogs/v1/modules///manifest.json` describes one + published module version, including repository refs, package refs, contracts, + compatibility windows, signatures, and available artifacts. +- `/catalogs/v1/modules//index.json` lists available versions for + one module. +- `/catalogs/v1/modules/index.json` lists all published modules. + +Gitea tags/releases remain the source release anchors. The public GovOPlaN +catalog directory becomes the installation-facing release repository that points +to those anchors and carries structured compatibility information. + +Selective catalog candidates now include this directory tree. Publishing a +candidate copies the channel catalog, keyring, and `modules/` tree into the +website repository together. + +Current limitation: selective catalog generation updates existing catalog +entries. Initial catalog entry synthesis for brand-new modules is still a +separate backend slice. + +## Direction + +The console should grow in slices: + +1. Read-only dashboard and next-action suggestions. +2. Release plan builder that writes explicit JSON plans. +3. Dry-run executor that shows exact commands and expected file changes. +4. Apply executor for commit, tag, push, artifact build, signing, and catalog + publication. +5. Compatibility planner for manifest contracts and version ranges. +6. Install/test workflow that validates a release from tags or catalog entries. + +All mutation must stay explicit: plan first, dry run second, apply only after a +clear confirmation. diff --git a/docs/REPOSITORY_INDEX.md b/docs/REPOSITORY_INDEX.md new file mode 100644 index 0000000..6940365 --- /dev/null +++ b/docs/REPOSITORY_INDEX.md @@ -0,0 +1,89 @@ +# GovOPlaN Repository Index + +Generated from `repositories.json`. Use that JSON file as the machine-readable source of truth; this page is the human-readable link index. + +## System + +| Repository | Subtype | Local path | Gitea | +| --- | --- | --- | --- | +| `govoplan` | `meta` | `../govoplan` | [govoplan](https://git.add-ideas.de/add-ideas/govoplan) | +| `govoplan-core` | `kernel` | `../govoplan-core` | [govoplan-core](https://git.add-ideas.de/add-ideas/govoplan-core) | + +## Module + +| Repository | Subtype | Local path | Gitea | +| --- | --- | --- | --- | +| `govoplan-access` | `platform` | `../govoplan-access` | [govoplan-access](https://git.add-ideas.de/add-ideas/govoplan-access) | +| `govoplan-addresses` | `domain` | `../govoplan-addresses` | [govoplan-addresses](https://git.add-ideas.de/add-ideas/govoplan-addresses) | +| `govoplan-admin` | `platform` | `../govoplan-admin` | [govoplan-admin](https://git.add-ideas.de/add-ideas/govoplan-admin) | +| `govoplan-appointments` | `domain` | `../govoplan-appointments` | [govoplan-appointments](https://git.add-ideas.de/add-ideas/govoplan-appointments) | +| `govoplan-approvals` | `domain` | `../govoplan-approvals` | [govoplan-approvals](https://git.add-ideas.de/add-ideas/govoplan-approvals) | +| `govoplan-assets` | `domain` | `../govoplan-assets` | [govoplan-assets](https://git.add-ideas.de/add-ideas/govoplan-assets) | +| `govoplan-audit` | `platform` | `../govoplan-audit` | [govoplan-audit](https://git.add-ideas.de/add-ideas/govoplan-audit) | +| `govoplan-booking` | `domain` | `../govoplan-booking` | [govoplan-booking](https://git.add-ideas.de/add-ideas/govoplan-booking) | +| `govoplan-calendar` | `domain` | `../govoplan-calendar` | [govoplan-calendar](https://git.add-ideas.de/add-ideas/govoplan-calendar) | +| `govoplan-campaign` | `domain` | `../govoplan-campaign` | [govoplan-campaign](https://git.add-ideas.de/add-ideas/govoplan-campaign) | +| `govoplan-cases` | `domain` | `../govoplan-cases` | [govoplan-cases](https://git.add-ideas.de/add-ideas/govoplan-cases) | +| `govoplan-certificates` | `domain` | `../govoplan-certificates` | [govoplan-certificates](https://git.add-ideas.de/add-ideas/govoplan-certificates) | +| `govoplan-committee` | `domain` | `../govoplan-committee` | [govoplan-committee](https://git.add-ideas.de/add-ideas/govoplan-committee) | +| `govoplan-consultation` | `domain` | `../govoplan-consultation` | [govoplan-consultation](https://git.add-ideas.de/add-ideas/govoplan-consultation) | +| `govoplan-contracts` | `domain` | `../govoplan-contracts` | [govoplan-contracts](https://git.add-ideas.de/add-ideas/govoplan-contracts) | +| `govoplan-dashboard` | `platform` | `../govoplan-dashboard` | [govoplan-dashboard](https://git.add-ideas.de/add-ideas/govoplan-dashboard) | +| `govoplan-dms` | `domain` | `../govoplan-dms` | [govoplan-dms](https://git.add-ideas.de/add-ideas/govoplan-dms) | +| `govoplan-dist-lists` | `domain` | `../govoplan-dist-lists` | [govoplan-dist-lists](https://git.add-ideas.de/add-ideas/govoplan-dist-lists) | +| `govoplan-docs` | `platform` | `../govoplan-docs` | [govoplan-docs](https://git.add-ideas.de/add-ideas/govoplan-docs) | +| `govoplan-erp` | `domain` | `../govoplan-erp` | [govoplan-erp](https://git.add-ideas.de/add-ideas/govoplan-erp) | +| `govoplan-evaluation` | `domain` | `../govoplan-evaluation` | [govoplan-evaluation](https://git.add-ideas.de/add-ideas/govoplan-evaluation) | +| `govoplan-facilities` | `domain` | `../govoplan-facilities` | [govoplan-facilities](https://git.add-ideas.de/add-ideas/govoplan-facilities) | +| `govoplan-files` | `domain` | `../govoplan-files` | [govoplan-files](https://git.add-ideas.de/add-ideas/govoplan-files) | +| `govoplan-forms` | `domain` | `../govoplan-forms` | [govoplan-forms](https://git.add-ideas.de/add-ideas/govoplan-forms) | +| `govoplan-forms-runtime` | `platform` | `../govoplan-forms-runtime` | [govoplan-forms-runtime](https://git.add-ideas.de/add-ideas/govoplan-forms-runtime) | +| `govoplan-grants` | `domain` | `../govoplan-grants` | [govoplan-grants](https://git.add-ideas.de/add-ideas/govoplan-grants) | +| `govoplan-helpdesk` | `domain` | `../govoplan-helpdesk` | [govoplan-helpdesk](https://git.add-ideas.de/add-ideas/govoplan-helpdesk) | +| `govoplan-identity` | `platform` | `../govoplan-identity` | [govoplan-identity](https://git.add-ideas.de/add-ideas/govoplan-identity) | +| `govoplan-identity-trust` | `platform` | `../govoplan-identity-trust` | [govoplan-identity-trust](https://git.add-ideas.de/add-ideas/govoplan-identity-trust) | +| `govoplan-idm` | `platform` | `../govoplan-idm` | [govoplan-idm](https://git.add-ideas.de/add-ideas/govoplan-idm) | +| `govoplan-inspections` | `domain` | `../govoplan-inspections` | [govoplan-inspections](https://git.add-ideas.de/add-ideas/govoplan-inspections) | +| `govoplan-issue-reporting` | `domain` | `../govoplan-issue-reporting` | [govoplan-issue-reporting](https://git.add-ideas.de/add-ideas/govoplan-issue-reporting) | +| `govoplan-learning` | `domain` | `../govoplan-learning` | [govoplan-learning](https://git.add-ideas.de/add-ideas/govoplan-learning) | +| `govoplan-ledger` | `domain` | `../govoplan-ledger` | [govoplan-ledger](https://git.add-ideas.de/add-ideas/govoplan-ledger) | +| `govoplan-mail` | `domain` | `../govoplan-mail` | [govoplan-mail](https://git.add-ideas.de/add-ideas/govoplan-mail) | +| `govoplan-notifications` | `platform` | `../govoplan-notifications` | [govoplan-notifications](https://git.add-ideas.de/add-ideas/govoplan-notifications) | +| `govoplan-ops` | `platform` | `../govoplan-ops` | [govoplan-ops](https://git.add-ideas.de/add-ideas/govoplan-ops) | +| `govoplan-organizations` | `platform` | `../govoplan-organizations` | [govoplan-organizations](https://git.add-ideas.de/add-ideas/govoplan-organizations) | +| `govoplan-payments` | `domain` | `../govoplan-payments` | [govoplan-payments](https://git.add-ideas.de/add-ideas/govoplan-payments) | +| `govoplan-permits` | `domain` | `../govoplan-permits` | [govoplan-permits](https://git.add-ideas.de/add-ideas/govoplan-permits) | +| `govoplan-policy` | `platform` | `../govoplan-policy` | [govoplan-policy](https://git.add-ideas.de/add-ideas/govoplan-policy) | +| `govoplan-poll` | `domain` | `../govoplan-poll` | [govoplan-poll](https://git.add-ideas.de/add-ideas/govoplan-poll) | +| `govoplan-portal` | `domain` | `../govoplan-portal` | [govoplan-portal](https://git.add-ideas.de/add-ideas/govoplan-portal) | +| `govoplan-postbox` | `domain` | `../govoplan-postbox` | [govoplan-postbox](https://git.add-ideas.de/add-ideas/govoplan-postbox) | +| `govoplan-procurement` | `domain` | `../govoplan-procurement` | [govoplan-procurement](https://git.add-ideas.de/add-ideas/govoplan-procurement) | +| `govoplan-records` | `domain` | `../govoplan-records` | [govoplan-records](https://git.add-ideas.de/add-ideas/govoplan-records) | +| `govoplan-reporting` | `domain` | `../govoplan-reporting` | [govoplan-reporting](https://git.add-ideas.de/add-ideas/govoplan-reporting) | +| `govoplan-resources` | `domain` | `../govoplan-resources` | [govoplan-resources](https://git.add-ideas.de/add-ideas/govoplan-resources) | +| `govoplan-risk-compliance` | `domain` | `../govoplan-risk-compliance` | [govoplan-risk-compliance](https://git.add-ideas.de/add-ideas/govoplan-risk-compliance) | +| `govoplan-scheduling` | `domain` | `../govoplan-scheduling` | [govoplan-scheduling](https://git.add-ideas.de/add-ideas/govoplan-scheduling) | +| `govoplan-search` | `platform` | `../govoplan-search` | [govoplan-search](https://git.add-ideas.de/add-ideas/govoplan-search) | +| `govoplan-tasks` | `domain` | `../govoplan-tasks` | [govoplan-tasks](https://git.add-ideas.de/add-ideas/govoplan-tasks) | +| `govoplan-templates` | `domain` | `../govoplan-templates` | [govoplan-templates](https://git.add-ideas.de/add-ideas/govoplan-templates) | +| `govoplan-tenancy` | `platform` | `../govoplan-tenancy` | [govoplan-tenancy](https://git.add-ideas.de/add-ideas/govoplan-tenancy) | +| `govoplan-transparency` | `domain` | `../govoplan-transparency` | [govoplan-transparency](https://git.add-ideas.de/add-ideas/govoplan-transparency) | +| `govoplan-workflow` | `platform` | `../govoplan-workflow` | [govoplan-workflow](https://git.add-ideas.de/add-ideas/govoplan-workflow) | + +## Connector + +| Repository | Subtype | Local path | Gitea | +| --- | --- | --- | --- | +| `govoplan-connectors` | `connector-hub` | `../govoplan-connectors` | [govoplan-connectors](https://git.add-ideas.de/add-ideas/govoplan-connectors) | +| `govoplan-fit-connect` | `standard` | `../govoplan-fit-connect` | [govoplan-fit-connect](https://git.add-ideas.de/add-ideas/govoplan-fit-connect) | +| `govoplan-rest` | `protocol` | `../govoplan-rest` | [govoplan-rest](https://git.add-ideas.de/add-ideas/govoplan-rest) | +| `govoplan-soap` | `protocol` | `../govoplan-soap` | [govoplan-soap](https://git.add-ideas.de/add-ideas/govoplan-soap) | +| `govoplan-xoev` | `standard` | `../govoplan-xoev` | [govoplan-xoev](https://git.add-ideas.de/add-ideas/govoplan-xoev) | +| `govoplan-xrechnung` | `standard` | `../govoplan-xrechnung` | [govoplan-xrechnung](https://git.add-ideas.de/add-ideas/govoplan-xrechnung) | +| `govoplan-xta-osci` | `standard` | `../govoplan-xta-osci` | [govoplan-xta-osci](https://git.add-ideas.de/add-ideas/govoplan-xta-osci) | + +## Website + +| Repository | Subtype | Local path | Gitea | +| --- | --- | --- | --- | +| `addideas-govoplan-website` | `public-site` | `../addideas-govoplan-website` | [addideas-govoplan-website](https://git.add-ideas.de/add-ideas/addideas-govoplan-website) | diff --git a/docs/SECURITY_AUDIT.md b/docs/SECURITY_AUDIT.md index dad1a04..fd1bfda 100644 --- a/docs/SECURITY_AUDIT.md +++ b/docs/SECURITY_AUDIT.md @@ -40,6 +40,11 @@ The wrapper tags the toolbox image by a fingerprint of the Dockerfile and the existing local image instead of reinstalling all tools. The stable alias is `govoplan/security-audit:local` unless `SECURITY_AUDIT_IMAGE` is set. +Semgrep is installed separately in the toolbox image because current Semgrep +packages pin Click to an affected `8.1.x` line. The image upgrades Click after +Semgrep installation and fails during build if Semgrep no longer starts with the +patched Click version. + Force a cached rebuild: ```bash @@ -65,6 +70,13 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build - `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit. - `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon. +Bandit and Ruff security reports are split by source kind. Production code under +`src/` is written to `bandit.json` and `ruff-security.json` and controls strict +mode. Test code under `tests/` is still scanned for visibility, but its findings +are written separately to `bandit-tests.json` and `ruff-security-tests.json` so +fixture passwords, assertions, and temporary paths do not hide the production +baseline. + ## Gating The initial Gitea workflow runs in report-only mode: @@ -115,6 +127,13 @@ Keep active implementation status in issues instead of committing generated audit reports. `audit-reports/` is ignored; quote the report directory and the important scanner counts in the tracker issue. +The jscpd step is intentionally scoped to application and test source. It +excludes documentation snippets, package manifests, generated translations, +public SVG assets, workflow YAML, and declarative backend schema JSON because +those reports produce metadata or asset repetition rather than actionable source +duplication. Keep exclusions narrow and create child issues for source-code +clusters that cross module ownership or make behavior harder to change safely. + ## Image Freshness The regular `Security Audit` workflow reuses the fingerprinted toolbox image @@ -133,6 +152,8 @@ tools first: cd /mnt/DATA/git/govoplan python -m venv .venv ./.venv/bin/python -m pip install -r requirements-audit.txt +./.venv/bin/python -m pip install 'semgrep>=1.140,<2' +./.venv/bin/python -m pip install --upgrade --no-deps 'click>=8.3.3' ``` Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`) diff --git a/docs/gitea-labels.json b/docs/gitea-labels.json index f2908a0..2cadb17 100644 --- a/docs/gitea-labels.json +++ b/docs/gitea-labels.json @@ -149,6 +149,12 @@ "description": "GovOPlaN Dms module behavior or integration.", "exclusive": false }, + { + "name": "module/dist-lists", + "color": "0e8a16", + "description": "GovOPlaN Distribution Lists module behavior or integration.", + "exclusive": false + }, { "name": "module/erp", "color": "fef2c0", diff --git a/repositories.json b/repositories.json index 0a061de..ac8b8f3 100644 --- a/repositories.json +++ b/repositories.json @@ -23,6 +23,7 @@ {"name": "govoplan-contracts", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-contracts.git", "path": "govoplan-contracts"}, {"name": "govoplan-dashboard", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-dashboard.git", "path": "govoplan-dashboard"}, {"name": "govoplan-dms", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-dms.git", "path": "govoplan-dms"}, + {"name": "govoplan-dist-lists", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-dist-lists.git", "path": "govoplan-dist-lists"}, {"name": "govoplan-docs", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-docs.git", "path": "govoplan-docs"}, {"name": "govoplan-erp", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-erp.git", "path": "govoplan-erp"}, {"name": "govoplan-evaluation", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-evaluation.git", "path": "govoplan-evaluation"}, diff --git a/requirements-audit.txt b/requirements-audit.txt index 8e4870f..f2e9dda 100644 --- a/requirements-audit.txt +++ b/requirements-audit.txt @@ -1,4 +1,5 @@ bandit>=1.8,<2 +click>=8.3.3 filelock>=3.20.3 idna>=3.15 pip>=26.1.2 @@ -6,5 +7,4 @@ pip-audit>=2.9,<3 python-multipart>=0.0.31 radon>=6,<7 ruff>=0.14,<1 -semgrep>=1.140,<2 xenon>=0.9,<1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 301ada5..e58e34a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,10 +12,16 @@ -e ../govoplan-policy -e ../govoplan-audit -e ../govoplan-dashboard +-e ../govoplan-addresses +-e ../govoplan-dist-lists -e ../govoplan-files -e ../govoplan-mail -e ../govoplan-campaign -e ../govoplan-calendar +-e ../govoplan-poll +-e ../govoplan-scheduling +-e ../govoplan-notifications +-e ../govoplan-evaluation -e ../govoplan-docs -e ../govoplan-ops httpx==0.28.1 @@ -25,3 +31,4 @@ idna>=3.15 pip>=26.1.2 pip-audit>=2.9,<3 python-multipart>=0.0.31 +ruff>=0.14,<1 diff --git a/requirements-release.txt b/requirements-release.txt index 2e70e00..d0a77e7 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -11,9 +11,14 @@ govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0. govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.8 govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.8 govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.8 +govoplan-addresses @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-addresses.git@v0.1.8 govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.8 govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.8 govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.8 govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.8 +govoplan-poll @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-poll.git@v0.1.8 +govoplan-scheduling @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-scheduling.git@v0.1.8 +govoplan-notifications @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-notifications.git@v0.1.8 +govoplan-evaluation @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-evaluation.git@v0.1.8 govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.8 govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.8 diff --git a/tools/checks/check-contracts.py b/tools/checks/check-contracts.py new file mode 100644 index 0000000..ea986db --- /dev/null +++ b/tools/checks/check-contracts.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Static cross-repository module contract check.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +META_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(META_ROOT / "tools" / "release")) + +from govoplan_release.contracts import ( # noqa: E402 + collect_contracts, + format_version_range, + interface_impact_map, + provided_interface_map, + validate_contracts, +) +from govoplan_release.model import RepositorySnapshot, to_jsonable # noqa: E402 +from govoplan_release.workspace import load_repository_specs, resolve_repo_path, resolve_workspace_root # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--workspace-root", + type=Path, + default=None, + help="Workspace root containing the repositories. Defaults to repositories.json default_parent.", + ) + parser.add_argument("--include-website", action="store_true", help="Also scan website repositories.") + parser.add_argument("--json", action="store_true", help="Print machine-readable JSON.") + parser.add_argument("--no-impact", action="store_true", help="Omit human-readable provider impact output.") + parser.add_argument("--strict-warnings", action="store_true", help="Fail when warnings are present.") + args = parser.parse_args() + + workspace_root = resolve_workspace_root(args.workspace_root) + repositories = repository_snapshots(workspace_root=workspace_root, include_website=args.include_website) + contracts = collect_contracts(repositories) + issues = validate_contracts(contracts) + providers = provided_interface_map(contracts) + consumers = interface_impact_map(contracts) + + payload = { + "workspace_root": str(workspace_root), + "repository_count": len(repositories), + "contract_count": len(contracts), + "provider_count": sum(len(contract.provides_interfaces) for contract in contracts), + "requirement_count": sum(len(contract.requires_interfaces) for contract in contracts), + "providers": provider_payload(providers, consumers), + "issues": to_jsonable(issues), + } + + blockers = [issue for issue in issues if issue.severity == "blocker"] + warnings = [issue for issue in issues if issue.severity == "warning"] + exit_code = 1 if blockers or (args.strict_warnings and warnings) else 0 + + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + return exit_code + + print("Static module contract scan") + print(f"Workspace: {workspace_root}") + print( + f"Contracts: {payload['contract_count']} modules, " + f"{payload['provider_count']} providers, {payload['requirement_count']} requirements" + ) + + if not args.no_impact: + print() + print("Provider impact:") + if providers: + for interface_name, provider_items in providers.items(): + provider_text = ", ".join( + f"{contract.module_id}@{provider.version} ({contract.repo})" + for contract, provider in provider_items + ) + consumer_text = ", ".join( + f"{contract.module_id} ({format_version_range(version_min=req.version_min, version_max_exclusive=req.version_max_exclusive)})" + for contract, req in consumers.get(interface_name, ()) + ) + if not consumer_text: + consumer_text = "no consumers" + print(f"- {interface_name}: {provider_text}; consumers: {consumer_text}") + else: + print("- no provided interfaces found") + + if issues: + print() + print("Contract issues:") + for issue in issues: + location = issue.repo or "workspace" + if issue.module_id: + location += f"/{issue.module_id}" + print(f"- [{issue.severity}] {issue.code}: {location}: {issue.message}") + else: + print() + print("Contract check passed.") + + return exit_code + + +def repository_snapshots(*, workspace_root: Path, include_website: bool) -> tuple[RepositorySnapshot, ...]: + snapshots: list[RepositorySnapshot] = [] + for spec in load_repository_specs(include_website=include_website): + path = resolve_repo_path(spec, workspace_root) + snapshots.append( + RepositorySnapshot( + spec=spec, + absolute_path=str(path), + exists=path.exists(), + is_git=(path / ".git").exists(), + ) + ) + return tuple(snapshots) + + +def provider_payload(providers, consumers): + payload: dict[str, dict[str, list[dict[str, str | bool | None]]]] = {} + for interface_name, provider_items in providers.items(): + payload[interface_name] = { + "providers": [ + { + "repo": contract.repo, + "module_id": contract.module_id, + "version": provider.version, + "manifest_path": contract.manifest_path, + } + for contract, provider in provider_items + ], + "consumers": [ + { + "repo": contract.repo, + "module_id": contract.module_id, + "version_min": requirement.version_min, + "version_max_exclusive": requirement.version_max_exclusive, + "optional": requirement.optional, + "manifest_path": contract.manifest_path, + } + for contract, requirement in consumers.get(interface_name, ()) + ], + } + return payload + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/checks/check-contracts.sh b/tools/checks/check-contracts.sh new file mode 100644 index 0000000..6bbf2a8 --- /dev/null +++ b/tools/checks/check-contracts.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}" +PYTHON="${PYTHON:-$VENV_ROOT/bin/python}" + +if [[ ! -x "$PYTHON" ]]; then + PYTHON=python3 +fi + +"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" "$@" diff --git a/tools/checks/check-dependency-hygiene.sh b/tools/checks/check-dependency-hygiene.sh index 20a3fd8..2112c1a 100644 --- a/tools/checks/check-dependency-hygiene.sh +++ b/tools/checks/check-dependency-hygiene.sh @@ -33,39 +33,6 @@ PY "$PYTHON" - <<'PY' from __future__ import annotations -import importlib.metadata -import pathlib -import sys - -prefix = pathlib.Path(sys.prefix) -legacy_patterns = ( - "__editable__.govoplan_module_multimailer-*.pth", - "__editable___govoplan_module_multimailer_*_finder.py", - "govoplan_module_multimailer-*.dist-info", -) -problems: list[pathlib.Path] = [] - -for site_packages in (prefix / "lib").glob("python*/site-packages"): - for pattern in legacy_patterns: - problems.extend(site_packages.glob(pattern)) - -for dist in importlib.metadata.distributions(): - if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer": - problems.append(pathlib.Path(str(dist.locate_file("")))) - -if problems: - print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr) - for path in sorted(set(problems)): - print(f" {path}", file=sys.stderr) - print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr) - raise SystemExit(1) - -print("No stale legacy multimailer package metadata found.") -PY - -"$PYTHON" - <<'PY' -from __future__ import annotations - import pathlib import sys @@ -75,6 +42,7 @@ scan_roots = [ root / "tests", root.parent / "govoplan-access" / "src", root.parent / "govoplan-admin" / "src", + root.parent / "govoplan-addresses" / "src", root.parent / "govoplan-audit" / "src", root.parent / "govoplan-calendar" / "src", root.parent / "govoplan-campaign" / "src", diff --git a/tools/checks/check-focused.sh b/tools/checks/check-focused.sh index c8a6257..d8a8fda 100644 --- a/tools/checks/check-focused.sh +++ b/tools/checks/check-focused.sh @@ -27,6 +27,7 @@ unset npm_config_tmp NPM_CONFIG_TMP cd "$ROOT" GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash "$META_ROOT/tools/checks/check-dependency-hygiene.sh" +"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact "$PYTHON" - <<'PY' import ast @@ -37,6 +38,7 @@ roots = [ pathlib.Path("/mnt/DATA/git/govoplan-core/src"), pathlib.Path("/mnt/DATA/git/govoplan-access/src"), pathlib.Path("/mnt/DATA/git/govoplan-admin/src"), + pathlib.Path("/mnt/DATA/git/govoplan-addresses/src"), pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"), pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"), pathlib.Path("/mnt/DATA/git/govoplan-identity/src"), @@ -67,7 +69,7 @@ if errors: print(f"AST syntax check passed for {count} Python files") PY -"$PYTHON" -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' +"$PYTHON" -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_addresses.backend.manifest; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' "$META_ROOT/tools/checks/check_dependency_boundaries.py" "$PYTHON" -m unittest tests.test_module_system "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests diff --git a/tools/checks/check-module-matrix.sh b/tools/checks/check-module-matrix.sh index 6dffa5c..73f6f6c 100644 --- a/tools/checks/check-module-matrix.sh +++ b/tools/checks/check-module-matrix.sh @@ -18,6 +18,7 @@ if [[ ! -x "$NPM" ]]; then fi cd "$ROOT" +"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact "$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts "$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py" diff --git a/tools/checks/check-release-integration.sh b/tools/checks/check-release-integration.sh index 109e1a0..73cba4b 100644 --- a/tools/checks/check-release-integration.sh +++ b/tools/checks/check-release-integration.sh @@ -92,6 +92,7 @@ install_cloned_webui_dependencies() { } run_step "Validate release refs and installed package metadata" +"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact "$PYTHON" - <<'PY' from __future__ import annotations diff --git a/tools/checks/check-security-audit.sh b/tools/checks/check-security-audit.sh index 4dc230e..b1f55f3 100644 --- a/tools/checks/check-security-audit.sh +++ b/tools/checks/check-security-audit.sh @@ -104,9 +104,17 @@ if [[ "${#REPOS[@]}" -le 12 ]]; then fi declare -a PY_ROOTS=() +declare -a PY_PROD_ROOTS=() +declare -a PY_TEST_ROOTS=() for repo in "${REPOS[@]}"; do - [[ -d "$repo/src" ]] && PY_ROOTS+=("$repo/src") - [[ -d "$repo/tests" ]] && PY_ROOTS+=("$repo/tests") + if [[ -d "$repo/src" ]]; then + PY_ROOTS+=("$repo/src") + PY_PROD_ROOTS+=("$repo/src") + fi + if [[ -d "$repo/tests" ]]; then + PY_ROOTS+=("$repo/tests") + PY_TEST_ROOTS+=("$repo/tests") + fi done safe_name() { @@ -184,13 +192,25 @@ run_semgrep() { } run_bandit() { - [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 - bandit -r "${PY_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" + local status=0 + if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then + bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" || status=1 + fi + if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then + bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" || true + fi + return "$status" } run_ruff_security() { - [[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0 - ruff check --select S --output-format json "${PY_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" + local status=0 + if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then + ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" || status=1 + fi + if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then + ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" || true + fi + return "$status" } run_gitleaks() { @@ -272,12 +292,39 @@ run_osv_scanner() { } run_jscpd() { + local ignored_paths + ignored_paths=( + "**/.git/**" + "**/node_modules/**" + "**/.venv/**" + "**/dist/**" + "**/build/**" + "**/audit-reports/**" + "**/package-lock.json" + "**/package-lock.release.json" + "**/package.json" + "package.json" + "**/generatedTranslations.ts" + "**/docs/gitea-labels.json" + "**/*.md" + "**/*.md:*" + "*.md" + "*.md:*" + "**/*.svg" + "*.svg" + ".gitea/workflows/*.yml" + "**/.gitea/workflows/*.yml" + "**/backend/schema/*.schema.json" + ) + local ignored_path_pattern + ignored_path_pattern="$(printf '%s,' "${ignored_paths[@]}")" + ignored_path_pattern="${ignored_path_pattern%,}" jscpd \ --silent \ --min-tokens 80 \ --reporters json \ --output "$REPORTS_DIR/jscpd" \ - --ignore "**/.git/**,**/node_modules/**,**/.venv/**,**/dist/**,**/build/**,**/audit-reports/**,**/package-lock.json,**/package-lock.release.json,**/generatedTranslations.ts,**/docs/gitea-labels.json" \ + --ignore "$ignored_path_pattern" \ "${REPOS[@]}" } diff --git a/tools/checks/check_dependency_boundaries.py b/tools/checks/check_dependency_boundaries.py index adce710..6adfd78 100644 --- a/tools/checks/check_dependency_boundaries.py +++ b/tools/checks/check_dependency_boundaries.py @@ -15,6 +15,7 @@ REPOS = { "core": CORE_ROOT / "src" / "govoplan_core", "access": REPOS_ROOT / "govoplan-access" / "src" / "govoplan_access", "admin": REPOS_ROOT / "govoplan-admin" / "src" / "govoplan_admin", + "addresses": REPOS_ROOT / "govoplan-addresses" / "src" / "govoplan_addresses", "tenancy": REPOS_ROOT / "govoplan-tenancy" / "src" / "govoplan_tenancy", "organizations": REPOS_ROOT / "govoplan-organizations" / "src" / "govoplan_organizations", "identity": REPOS_ROOT / "govoplan-identity" / "src" / "govoplan_identity", @@ -29,6 +30,7 @@ PREFIXES = { "core": "govoplan_core", "access": "govoplan_access", "admin": "govoplan_admin", + "addresses": "govoplan_addresses", "tenancy": "govoplan_tenancy", "organizations": "govoplan_organizations", "identity": "govoplan_identity", @@ -39,16 +41,18 @@ PREFIXES = { "mail": "govoplan_mail", "campaign": "govoplan_campaign", } -FEATURE_OWNERS = ("files", "mail", "campaign") +FEATURE_OWNERS = ("addresses", "files", "mail", "campaign") PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard") WEBUI_REPOS = { "core": CORE_ROOT / "webui", + "addresses": REPOS_ROOT / "govoplan-addresses" / "webui", "files": REPOS_ROOT / "govoplan-files" / "webui", "mail": REPOS_ROOT / "govoplan-mail" / "webui", "campaign": REPOS_ROOT / "govoplan-campaign" / "webui", } WEBUI_PACKAGES = { "core": "@govoplan/core-webui", + "addresses": "@govoplan/addresses-webui", "files": "@govoplan/files-webui", "mail": "@govoplan/mail-webui", "campaign": "@govoplan/campaign-webui", diff --git a/tools/checks/security-audit/Dockerfile b/tools/checks/security-audit/Dockerfile index 4cf815c..d066a2f 100644 --- a/tools/checks/security-audit/Dockerfile +++ b/tools/checks/security-audit/Dockerfile @@ -34,6 +34,8 @@ COPY requirements-audit.txt /tmp/requirements-audit.txt RUN python -m pip install --upgrade pip \ && python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \ + && python -m pip install --no-cache-dir "semgrep>=1.140,<2" \ + && python -m pip install --no-cache-dir --upgrade --no-deps "click>=8.3.3" \ && npm install -g jscpd \ && semgrep --version \ && bandit --version \ diff --git a/tools/generate-release-catalog.py b/tools/generate-release-catalog.py new file mode 100644 index 0000000..7073718 --- /dev/null +++ b/tools/generate-release-catalog.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Compatibility wrapper for the release catalog generator.""" + +from __future__ import annotations + +from pathlib import Path +import runpy + +runpy.run_path(str(Path(__file__).resolve().parent / "release" / "generate-release-catalog.py"), run_name="__main__") diff --git a/tools/gitea/gitea_common.py b/tools/gitea/gitea_common.py index 63c0031..16daa7d 100644 --- a/tools/gitea/gitea_common.py +++ b/tools/gitea/gitea_common.py @@ -159,7 +159,7 @@ class GiteaClient: if not host: raise GiteaError("Gitea API URL is missing a hostname.") if self._api_url.scheme == "https": - self._connection = http.client.HTTPSConnection(host, self._api_url.port, timeout=self.timeout) + self._connection = http.client.HTTPSConnection(host, self._api_url.port, timeout=self.timeout) # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected elif self._api_url.scheme == "http": self._connection = http.client.HTTPConnection(host, self._api_url.port, timeout=self.timeout) else: diff --git a/tools/launch/launch-dev.sh b/tools/launch/launch-dev.sh index 9c11b73..ad2fcaf 100644 --- a/tools/launch/launch-dev.sh +++ b/tools/launch/launch-dev.sh @@ -20,6 +20,7 @@ FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}" FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}" FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" +AUTO_SYNC_PYTHON="${GOVOPLAN_AUTO_SYNC_PYTHON:-1}" DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}" LOG_DIR="${GOVOPLAN_DEV_LOG_DIR:-$META_ROOT/runtime/dev-launcher}" @@ -137,6 +138,14 @@ trap cleanup EXIT INT TERM [ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json" [ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install" +if [ "$AUTO_SYNC_PYTHON" = "1" ]; then + "$PYTHON" "$META_ROOT/tools/repo/sync-python-environment.py" \ + --requirements "$META_ROOT/requirements-dev.txt" \ + --python "$PYTHON" \ + --stamp "$VENV_ROOT/.govoplan-sync-requirements-dev.json" \ + || fail "Python environment synchronization failed" +fi + ( cd "$META_ROOT" "$PYTHON" - <<'PY' diff --git a/tools/launch/launch-production-like-dev.sh b/tools/launch/launch-production-like-dev.sh index e585351..bc3b994 100644 --- a/tools/launch/launch-production-like-dev.sh +++ b/tools/launch/launch-production-like-dev.sh @@ -49,7 +49,7 @@ set +a export APP_ENV="${APP_ENV:-staging}" export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}" -export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}" +export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,notifications,docs,ops}" export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}" diff --git a/tools/launch/production-like-dev.sh b/tools/launch/production-like-dev.sh index 9b5900f..12bab81 100644 --- a/tools/launch/production-like-dev.sh +++ b/tools/launch/production-like-dev.sh @@ -66,7 +66,7 @@ export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-po export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}" export CELERY_ENABLED="${CELERY_ENABLED:-true}" -export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}" +export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,notifications,docs,ops}" export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}" export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$META_ROOT/runtime/production-like/files}" export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}" diff --git a/tools/release/generate-release-catalog.py b/tools/release/generate-release-catalog.py index 09f4428..a15575a 100644 --- a/tools/release/generate-release-catalog.py +++ b/tools/release/generate-release-catalog.py @@ -110,6 +110,15 @@ CATALOG_MODULES = ( tags=("official", "platform-module"), webui_package="@govoplan/dashboard-webui", ), + CatalogModule( + module_id="addresses", + repo="govoplan-addresses", + python_package="govoplan-addresses", + name="Addresses", + description="Reusable address directories, recipient sources, consent metadata, and address quality workflows.", + tags=("official", "business-module"), + webui_package="@govoplan/addresses-webui", + ), CatalogModule( module_id="files", repo="govoplan-files", diff --git a/tools/release/govoplan_release/__init__.py b/tools/release/govoplan_release/__init__.py new file mode 100644 index 0000000..3d11806 --- /dev/null +++ b/tools/release/govoplan_release/__init__.py @@ -0,0 +1,67 @@ +"""Shared release tooling for the local GovOPlaN release console.""" + +from __future__ import annotations + +from .model import ReleaseDashboard, ReleasePlan, RepositorySnapshot, RepositorySpec + +__all__ = [ + "ReleaseDashboard", + "ReleasePlan", + "RepositorySnapshot", + "RepositorySpec", + "build_dashboard", + "build_release_intelligence", + "build_release_plan", + "build_selective_catalog_candidate", + "build_selective_release_plan", + "publish_catalog_candidate", + "prepare_repositories", + "push_repositories", + "sync_repositories", +] + + +def __getattr__(name: str): + value = _load_lazy_export(name) + globals()[name] = value + return value + + +def _load_lazy_export(name: str): + if name == "build_dashboard": + from .dashboard import build_dashboard + + return build_dashboard + if name == "build_release_intelligence": + from .release_intelligence import build_release_intelligence + + return build_release_intelligence + if name == "build_release_plan": + from .planner import build_release_plan + + return build_release_plan + if name == "build_selective_catalog_candidate": + from .selective_catalog import build_selective_catalog_candidate + + return build_selective_catalog_candidate + if name == "build_selective_release_plan": + from .selective_planner import build_selective_release_plan + + return build_selective_release_plan + if name == "publish_catalog_candidate": + from .publisher import publish_catalog_candidate + + return publish_catalog_candidate + if name == "prepare_repositories": + from .repository_prepare import prepare_repositories + + return prepare_repositories + if name == "push_repositories": + from .repository_push import push_repositories + + return push_repositories + if name == "sync_repositories": + from .repository_sync import sync_repositories + + return sync_repositories + raise AttributeError(name) diff --git a/tools/release/govoplan_release/catalog.py b/tools/release/govoplan_release/catalog.py new file mode 100644 index 0000000..8e579a4 --- /dev/null +++ b/tools/release/govoplan_release/catalog.py @@ -0,0 +1,185 @@ +"""Release catalog inspection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from urllib.error import HTTPError, URLError + +from .http_fetch import fetch_http +from .model import CatalogSnapshot +from .workspace import website_root + +DEFAULT_PUBLIC_BASE_URL = "https://govoplan.add-ideas.de" + + +def collect_catalog_snapshot( + *, + workspace_root: Path, + channel: str = "stable", + public_base_url: str = DEFAULT_PUBLIC_BASE_URL, + check_public: bool = False, +) -> CatalogSnapshot: + web_root = website_root(workspace_root) + catalog_path = web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json" + keyring_path = web_root / "public" / "catalogs" / "v1" / "keyring.json" + public_catalog_url = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json" + public_keyring_url = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json" + + catalog_version: str | None = None + core_release_version: str | None = None + module_count = 0 + signature_count = 0 + generated_at: str | None = None + expires_at: str | None = None + catalog_error: str | None = None + local_catalog_hash: str | None = None + + if catalog_path.exists(): + try: + payload = json.loads(catalog_path.read_text(encoding="utf-8")) + local_catalog_hash = canonical_hash(payload) + parsed = parse_catalog_payload(payload) + catalog_version = parsed["catalog_version"] + core_release_version = parsed["core_release_version"] + module_count = parsed["module_count"] or 0 + signature_count = parsed["signature_count"] or 0 + generated_at = parsed["generated_at"] + expires_at = parsed["expires_at"] + except Exception as exc: # noqa: BLE001 - shown in local operator UI. + catalog_error = str(exc) + + key_count = 0 + keyring_error: str | None = None + local_keyring_hash: str | None = None + if keyring_path.exists(): + try: + payload = json.loads(keyring_path.read_text(encoding="utf-8")) + local_keyring_hash = canonical_hash(payload) + keys = payload.get("keys") if isinstance(payload, dict) else None + key_count = len(keys) if isinstance(keys, list) else 0 + except Exception as exc: # noqa: BLE001 - shown in local operator UI. + keyring_error = str(exc) + + public_catalog_exists: bool | None = None + public_keyring_exists: bool | None = None + public_core_release_version: str | None = None + public_module_count: int | None = None + public_signature_count: int | None = None + public_key_count: int | None = None + public_catalog_hash: str | None = None + public_keyring_hash: str | None = None + public_catalog_error: str | None = None + public_keyring_error: str | None = None + + if check_public: + public_catalog = fetch_json(public_catalog_url) + if public_catalog["ok"]: + public_catalog_exists = True + payload = public_catalog["payload"] + public_catalog_hash = canonical_hash(payload) + parsed = parse_catalog_payload(payload) + public_core_release_version = parsed["core_release_version"] + public_module_count = parsed["module_count"] + public_signature_count = parsed["signature_count"] + else: + public_catalog_exists = False + public_catalog_error = str(public_catalog["error"]) + + public_keyring = fetch_json(public_keyring_url) + if public_keyring["ok"]: + public_keyring_exists = True + payload = public_keyring["payload"] + public_keyring_hash = canonical_hash(payload) + keys = payload.get("keys") if isinstance(payload, dict) else None + public_key_count = len(keys) if isinstance(keys, list) else 0 + else: + public_keyring_exists = False + public_keyring_error = str(public_keyring["error"]) + + return CatalogSnapshot( + channel=channel, + website_path=str(web_root), + catalog_path=str(catalog_path), + catalog_exists=catalog_path.exists(), + keyring_path=str(keyring_path), + keyring_exists=keyring_path.exists(), + catalog_version=catalog_version, + core_release_version=core_release_version, + module_count=module_count, + signature_count=signature_count, + key_count=key_count, + generated_at=generated_at, + expires_at=expires_at, + catalog_error=catalog_error, + keyring_error=keyring_error, + local_catalog_hash=local_catalog_hash, + local_keyring_hash=local_keyring_hash, + public_catalog_url=public_catalog_url, + public_keyring_url=public_keyring_url, + public_checked=check_public, + public_catalog_exists=public_catalog_exists, + public_keyring_exists=public_keyring_exists, + public_core_release_version=public_core_release_version, + public_module_count=public_module_count, + public_signature_count=public_signature_count, + public_key_count=public_key_count, + public_catalog_hash=public_catalog_hash, + public_keyring_hash=public_keyring_hash, + public_catalog_error=public_catalog_error, + public_keyring_error=public_keyring_error, + catalog_matches_public=compare_hashes(local_catalog_hash, public_catalog_hash) if check_public else None, + keyring_matches_public=compare_hashes(local_keyring_hash, public_keyring_hash) if check_public else None, + ) + + +def parse_catalog_payload(payload: object) -> dict[str, str | int | None]: + if not isinstance(payload, dict): + return { + "catalog_version": None, + "core_release_version": None, + "module_count": 0, + "signature_count": 0, + "generated_at": None, + "expires_at": None, + } + core_release = payload.get("core_release") + modules = payload.get("modules") + signatures = payload.get("signatures") + return { + "catalog_version": string_or_none(payload.get("catalog_version")), + "core_release_version": string_or_none(core_release.get("version")) if isinstance(core_release, dict) else None, + "module_count": len(modules) if isinstance(modules, list) else 0, + "signature_count": len(signatures) if isinstance(signatures, list) else 0, + "generated_at": string_or_none(payload.get("generated_at")), + "expires_at": string_or_none(payload.get("expires_at")), + } + + +def fetch_json(url: str) -> dict[str, object]: + try: + response = fetch_http( + url, + timeout=10, + label="Public release catalog URL", + headers={"User-Agent": "GovOPlaN release-console"}, + ) + return {"ok": True, "payload": json.loads(response.text())} + except (HTTPError, URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc: + return {"ok": False, "error": str(exc)} + + +def canonical_hash(payload: object) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def compare_hashes(local_hash: str | None, public_hash: str | None) -> bool | None: + if local_hash is None or public_hash is None: + return None + return local_hash == public_hash + + +def string_or_none(value: object) -> str | None: + return value if isinstance(value, str) else None diff --git a/tools/release/govoplan_release/contracts.py b/tools/release/govoplan_release/contracts.py new file mode 100644 index 0000000..7ab1621 --- /dev/null +++ b/tools/release/govoplan_release/contracts.py @@ -0,0 +1,445 @@ +"""Manifest contract extraction and validation for release planning. + +The static scanner intentionally parses manifest source files with ``ast`` +instead of importing module packages. That keeps contract checks usable before +``pip install`` has been rerun for a changed ``pyproject.toml`` or entry point. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +from .model import ( + CompatibilityIssue, + InterfaceProviderSnapshot, + InterfaceRequirementSnapshot, + ModuleContractSnapshot, + RepositorySnapshot, +) + + +def collect_contracts(repositories: tuple[RepositorySnapshot, ...]) -> tuple[ModuleContractSnapshot, ...]: + contracts: list[ModuleContractSnapshot] = [] + for repo in repositories: + if not repo.exists or not repo.is_git: + continue + root = Path(repo.absolute_path) + for manifest in sorted(root.glob("src/**/backend/manifest.py")): + contract = parse_manifest_contract(manifest, repo_name=repo.spec.name) + if contract is not None: + contracts.append(contract) + return tuple(contracts) + + +def validate_contracts(contracts: tuple[ModuleContractSnapshot, ...]) -> tuple[CompatibilityIssue, ...]: + """Validate the statically collected module interface graph. + + This mirrors the runtime interface closure validation in + ``govoplan_core.core.registry`` but reports all issues at once for CI and + release planning. + """ + + issues: list[CompatibilityIssue] = [] + providers: dict[str, list[tuple[ModuleContractSnapshot, InterfaceProviderSnapshot]]] = {} + module_ids: dict[str, list[ModuleContractSnapshot]] = {} + + for contract in contracts: + module_ids.setdefault(contract.module_id, []).append(contract) + local_provider_names: set[str] = set() + local_requirement_names: set[str] = set() + + for provider in contract.provides_interfaces: + if provider.name in local_provider_names: + issues.append( + CompatibilityIssue( + severity="blocker", + code="duplicate_provided_interface", + message=f"{contract.module_id} provides interface {provider.name!r} more than once.", + repo=contract.repo, + module_id=contract.module_id, + interface=provider.name, + ) + ) + local_provider_names.add(provider.name) + if not valid_interface_name(provider.name): + issues.append( + CompatibilityIssue( + severity="blocker", + code="invalid_provided_interface_name", + message=f"{contract.module_id} provides invalid interface name {provider.name!r}.", + repo=contract.repo, + module_id=contract.module_id, + interface=provider.name, + ) + ) + if not provider.version.strip(): + issues.append( + CompatibilityIssue( + severity="blocker", + code="missing_provided_interface_version", + message=f"{contract.module_id} provides interface {provider.name!r} without a version.", + repo=contract.repo, + module_id=contract.module_id, + interface=provider.name, + ) + ) + providers.setdefault(provider.name, []).append((contract, provider)) + + for requirement in contract.requires_interfaces: + if requirement.name in local_requirement_names: + issues.append( + CompatibilityIssue( + severity="blocker", + code="duplicate_required_interface", + message=f"{contract.module_id} requires interface {requirement.name!r} more than once.", + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + local_requirement_names.add(requirement.name) + if not valid_interface_name(requirement.name): + issues.append( + CompatibilityIssue( + severity="blocker", + code="invalid_required_interface_name", + message=f"{contract.module_id} requires invalid interface name {requirement.name!r}.", + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + if requirement.version_min is not None and not requirement.version_min.strip(): + issues.append( + CompatibilityIssue( + severity="blocker", + code="blank_required_interface_minimum", + message=f"{contract.module_id} has a blank minimum version for {requirement.name!r}.", + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + if requirement.version_max_exclusive is not None and not requirement.version_max_exclusive.strip(): + issues.append( + CompatibilityIssue( + severity="blocker", + code="blank_required_interface_maximum", + message=f"{contract.module_id} has a blank maximum version for {requirement.name!r}.", + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + if not version_range_is_valid( + version_min=requirement.version_min, + version_max_exclusive=requirement.version_max_exclusive, + ): + issues.append( + CompatibilityIssue( + severity="blocker", + code="invalid_required_interface_range", + message=( + f"{contract.module_id} requires {requirement.name!r} with invalid range " + f"{format_version_range(version_min=requirement.version_min, version_max_exclusive=requirement.version_max_exclusive)}." + ), + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + + for module_id, matches in sorted(module_ids.items()): + if len(matches) > 1: + repos = ", ".join(sorted(contract.repo for contract in matches)) + for contract in matches: + issues.append( + CompatibilityIssue( + severity="blocker", + code="duplicate_module_id", + message=f"Module id {module_id!r} is declared by multiple repositories: {repos}.", + repo=contract.repo, + module_id=module_id, + ) + ) + + for contract in contracts: + for requirement in contract.requires_interfaces: + available = providers.get(requirement.name, []) + matching = [ + (provider_contract, provider) + for provider_contract, provider in available + if version_satisfies_range( + provider.version, + version_min=requirement.version_min, + version_max_exclusive=requirement.version_max_exclusive, + ) + ] + if matching: + continue + version_range = format_version_range( + version_min=requirement.version_min, + version_max_exclusive=requirement.version_max_exclusive, + ) + if not available: + if requirement.optional: + continue + issues.append( + CompatibilityIssue( + severity="blocker", + code="required_interface_missing", + message=f"{contract.module_id} requires unavailable interface {requirement.name!r} ({version_range}).", + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + continue + available_versions = ", ".join( + f"{provider_contract.module_id}@{provider.version}" for provider_contract, provider in available + ) + issues.append( + CompatibilityIssue( + severity="warning" if requirement.optional else "blocker", + code="interface_version_mismatch", + message=( + f"{contract.module_id} requires interface {requirement.name!r} ({version_range}) " + f"but available providers are {available_versions}." + ), + repo=contract.repo, + module_id=contract.module_id, + interface=requirement.name, + ) + ) + + return tuple(sorted(issues, key=issue_sort_key)) + + +def interface_impact_map( + contracts: tuple[ModuleContractSnapshot, ...], +) -> dict[str, tuple[tuple[ModuleContractSnapshot, InterfaceRequirementSnapshot], ...]]: + consumers: dict[str, list[tuple[ModuleContractSnapshot, InterfaceRequirementSnapshot]]] = {} + for contract in contracts: + for requirement in contract.requires_interfaces: + consumers.setdefault(requirement.name, []).append((contract, requirement)) + return {name: tuple(items) for name, items in sorted(consumers.items())} + + +def provided_interface_map( + contracts: tuple[ModuleContractSnapshot, ...], +) -> dict[str, tuple[tuple[ModuleContractSnapshot, InterfaceProviderSnapshot], ...]]: + providers: dict[str, list[tuple[ModuleContractSnapshot, InterfaceProviderSnapshot]]] = {} + for contract in contracts: + for provider in contract.provides_interfaces: + providers.setdefault(provider.name, []).append((contract, provider)) + return {name: tuple(items) for name, items in sorted(providers.items())} + + +def parse_manifest_contract(path: Path, *, repo_name: str) -> ModuleContractSnapshot | None: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + constants = imported_string_constants(tree, path) + constants.update(module_constants(tree, initial=constants)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if call_name(node.func) != "ModuleManifest": + continue + keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg} + module_id = value_as_string(keywords.get("id"), constants) + if module_id is None: + continue + return ModuleContractSnapshot( + repo=repo_name, + module_id=module_id, + module_version=value_as_string(keywords.get("version"), constants), + manifest_path=str(path), + provides_interfaces=parse_providers(keywords.get("provides_interfaces"), constants), + requires_interfaces=parse_requirements(keywords.get("requires_interfaces"), constants), + ) + return None + + +def parse_providers(node: ast.AST | None, constants: dict[str, str]) -> tuple[InterfaceProviderSnapshot, ...]: + providers: list[InterfaceProviderSnapshot] = [] + for call in interface_calls(node, "ModuleInterfaceProvider"): + keywords = {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg} + name = value_as_string(keywords.get("name"), constants) + version = value_as_string(keywords.get("version"), constants) + if name and version: + providers.append(InterfaceProviderSnapshot(name=name, version=version)) + return tuple(providers) + + +def parse_requirements(node: ast.AST | None, constants: dict[str, str]) -> tuple[InterfaceRequirementSnapshot, ...]: + requirements: list[InterfaceRequirementSnapshot] = [] + for call in interface_calls(node, "ModuleInterfaceRequirement"): + keywords = {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg} + name = value_as_string(keywords.get("name"), constants) + if not name: + continue + requirements.append( + InterfaceRequirementSnapshot( + name=name, + version_min=value_as_string(keywords.get("version_min"), constants), + version_max_exclusive=value_as_string(keywords.get("version_max_exclusive"), constants), + optional=value_as_bool(keywords.get("optional")), + ) + ) + return tuple(requirements) + + +def interface_calls(node: ast.AST | None, expected_name: str) -> tuple[ast.Call, ...]: + if node is None: + return () + return tuple( + child + for child in ast.walk(node) + if isinstance(child, ast.Call) and call_name(child.func) == expected_name + ) + + +def imported_string_constants(tree: ast.Module, path: Path) -> dict[str, str]: + constants: dict[str, str] = {} + for node in tree.body: + if not isinstance(node, ast.ImportFrom): + continue + source = import_source_path(path, module=node.module, level=node.level) + if source is None or not source.exists(): + continue + try: + imported = module_constants(ast.parse(source.read_text(encoding="utf-8"), filename=str(source))) + except SyntaxError: + continue + for alias in node.names: + if alias.name in imported: + constants[alias.asname or alias.name] = imported[alias.name] + return constants + + +def import_source_path(path: Path, *, module: str | None, level: int) -> Path | None: + if level: + base = path.parent + for _ in range(level - 1): + base = base.parent + if module: + base = base.joinpath(*module.split(".")) + candidate = base.with_suffix(".py") + else: + src_root = next((parent for parent in path.parents if parent.name == "src"), None) + if src_root is None or module is None: + return None + candidate = src_root.joinpath(*module.split(".")).with_suffix(".py") + if candidate.exists(): + return candidate + init = candidate.with_suffix("") / "__init__.py" + return init if init.exists() else candidate + + +def module_constants(tree: ast.Module, *, initial: dict[str, str] | None = None) -> dict[str, str]: + constants: dict[str, str] = dict(initial or {}) + for node in tree.body: + if isinstance(node, ast.Assign): + value = node.value + for target in node.targets: + if isinstance(target, ast.Name): + resolved = value_as_string(value, constants) + if resolved is not None: + constants[target.id] = resolved + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + resolved = value_as_string(node.value, constants) + if resolved is not None: + constants[node.target.id] = resolved + return constants + + +def value_as_string(node: ast.AST | None, constants: dict[str, str]) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return constants.get(node.id) + return None + + +def value_as_bool(node: ast.AST | None) -> bool: + return bool(node.value) if isinstance(node, ast.Constant) and isinstance(node.value, bool) else False + + +def call_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return "" + + +def issue_sort_key(issue: CompatibilityIssue) -> tuple[int, str, str, str, str]: + severity_order = {"blocker": 0, "warning": 1, "info": 2} + return ( + severity_order.get(issue.severity, 9), + issue.repo or "", + issue.module_id or "", + issue.interface or "", + issue.code, + ) + + +def valid_interface_name(value: str) -> bool: + return bool(re.match(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$", value)) + + +def version_satisfies_range( + version: str, + *, + version_min: str | None = None, + version_max_exclusive: str | None = None, +) -> bool: + if version_min is not None and compare_versions(version, version_min) < 0: + return False + if version_max_exclusive is not None and compare_versions(version, version_max_exclusive) >= 0: + return False + return True + + +def version_range_is_valid( + *, + version_min: str | None = None, + version_max_exclusive: str | None = None, +) -> bool: + if version_min is None or version_max_exclusive is None: + return True + return compare_versions(version_min, version_max_exclusive) < 0 + + +def format_version_range( + *, + version_min: str | None = None, + version_max_exclusive: str | None = None, +) -> str: + parts: list[str] = [] + if version_min is not None: + parts.append(f">= {version_min}") + if version_max_exclusive is not None: + parts.append(f"< {version_max_exclusive}") + return ", ".join(parts) if parts else "any version" + + +def compare_versions(left: str, right: str) -> int: + left_parts = version_tuple(left) + right_parts = version_tuple(right) + max_length = max(len(left_parts), len(right_parts)) + padded_left = left_parts + (0,) * (max_length - len(left_parts)) + padded_right = right_parts + (0,) * (max_length - len(right_parts)) + if padded_left < padded_right: + return -1 + if padded_left > padded_right: + return 1 + return 0 + + +def version_tuple(value: str) -> tuple[int, ...]: + match = re.match(r"^\s*v?(\d+(?:\.\d+)*)", value) + if not match: + return (0,) + return tuple(int(part) for part in match.group(1).split(".")) diff --git a/tools/release/govoplan_release/dashboard.py b/tools/release/govoplan_release/dashboard.py new file mode 100644 index 0000000..0e3355b --- /dev/null +++ b/tools/release/govoplan_release/dashboard.py @@ -0,0 +1,88 @@ +"""Build read-only release dashboard snapshots.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +from .catalog import collect_catalog_snapshot +from .contracts import collect_contracts +from .git_state import collect_repository_snapshot, read_pyproject_version +from .migrations import collect_migration_snapshots +from .model import DashboardSummary, ReleaseDashboard, RepositorySnapshot +from .workspace import META_ROOT, core_root, load_repository_specs, resolve_workspace_root + + +def build_dashboard( + *, + workspace_root: Path | str | None = None, + target_version: str | None = None, + online: bool = False, + check_remote_tags: bool = False, + check_public_catalog: bool | None = None, + include_migrations: bool = False, + include_website: bool = False, + include_contracts: bool = True, + channel: str = "stable", +) -> ReleaseDashboard: + resolved_workspace = resolve_workspace_root(workspace_root) + if check_public_catalog is None: + check_public_catalog = online + if target_version is None: + target_version = read_pyproject_version(core_root(resolved_workspace)) + target_tag = f"v{target_version}" if target_version else None + + repositories = tuple( + collect_repository_snapshot(spec, workspace_root=resolved_workspace, target_tag=target_tag, online=check_remote_tags) + for spec in load_repository_specs(include_website=include_website) + ) + catalog = collect_catalog_snapshot(workspace_root=resolved_workspace, channel=channel, check_public=check_public_catalog) + migrations = collect_migration_snapshots() if include_migrations else () + contracts = collect_contracts(repositories) if include_contracts else () + return ReleaseDashboard( + generated_at=datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + meta_root=str(META_ROOT), + workspace_root=str(resolved_workspace), + target_version=target_version, + target_tag=target_tag, + online=online or check_remote_tags or check_public_catalog, + include_migrations=include_migrations, + summary=summarize(repositories, target_tag=target_tag), + repositories=repositories, + catalog=catalog, + migrations=migrations, + contracts=contracts, + ) + + +def summarize(repositories: tuple[RepositorySnapshot, ...], *, target_tag: str | None) -> DashboardSummary: + missing_count = sum(1 for repo in repositories if not repo.exists or not repo.is_git) + dirty_count = sum(1 for repo in repositories if repo.dirty) + ahead_count = sum(1 for repo in repositories if repo.ahead) + behind_count = sum(1 for repo in repositories if repo.behind) + no_head_count = sum(1 for repo in repositories if repo.exists and repo.is_git and not repo.has_head) + error_count = sum(1 for repo in repositories if repo.errors) + safe_directory_count = sum(1 for repo in repositories if repo.safe_directory_required) + local_target_tag_missing_count = ( + sum(1 for repo in repositories if repo.local_target_tag_exists is False and repo.versions.primary == target_tag.removeprefix("v")) + if target_tag + else 0 + ) + if missing_count or behind_count or safe_directory_count: + status = "blocked" + elif dirty_count or ahead_count or no_head_count or error_count or local_target_tag_missing_count: + status = "attention" + else: + status = "ready" + return DashboardSummary( + repository_count=len(repositories), + missing_count=missing_count, + dirty_count=dirty_count, + ahead_count=ahead_count, + behind_count=behind_count, + no_head_count=no_head_count, + error_count=error_count, + safe_directory_count=safe_directory_count, + local_target_tag_missing_count=local_target_tag_missing_count, + status=status, + ) diff --git a/tools/release/govoplan_release/git_state.py b/tools/release/govoplan_release/git_state.py new file mode 100644 index 0000000..88ded78 --- /dev/null +++ b/tools/release/govoplan_release/git_state.py @@ -0,0 +1,178 @@ +"""Read-only Git and version metadata collection.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re +import subprocess +import tomllib + +from .model import RepositorySnapshot, RepositorySpec, VersionSnapshot +from .workspace import resolve_repo_path + + +def collect_repository_snapshot( + spec: RepositorySpec, + *, + workspace_root: Path, + target_tag: str | None, + online: bool = False, +) -> RepositorySnapshot: + path = resolve_repo_path(spec, workspace_root) + if not path.exists(): + return RepositorySnapshot(spec=spec, absolute_path=str(path), exists=False, is_git=False, errors=("repository directory is missing",)) + if not (path / ".git").exists(): + return RepositorySnapshot(spec=spec, absolute_path=str(path), exists=True, is_git=False, errors=("not a git repository",)) + + errors: list[str] = [] + status_result = git(path, "status", "--porcelain", timeout=10) + if status_result.returncode != 0: + message = git_error(status_result) + if is_dubious_ownership_error(message): + return RepositorySnapshot( + spec=spec, + absolute_path=str(path), + exists=True, + is_git=True, + safe_directory_required=True, + safe_directory_command=safe_directory_command(path), + errors=(compact_git_error(message),), + ) + return RepositorySnapshot( + spec=spec, + absolute_path=str(path), + exists=True, + is_git=True, + errors=(compact_git_error(message),), + ) + + branch = git_text(path, "branch", "--show-current") + has_head = git_success(path, "rev-parse", "--verify", "HEAD") + head = git_text(path, "rev-parse", "--short", "HEAD") if has_head else None + upstream = git_text(path, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}") if has_head else None + ahead: int | None = None + behind: int | None = None + if upstream: + counts = git_text(path, "rev-list", "--left-right", "--count", "@{upstream}...HEAD") + parts = counts.split() + if len(parts) == 2 and all(part.isdigit() for part in parts): + behind = int(parts[0]) + ahead = int(parts[1]) + + local_target_tag_exists: bool | None = None + remote_target_tag_exists: bool | None = None + if target_tag: + local_target_tag_exists = git_success(path, "rev-parse", "-q", "--verify", f"refs/tags/{target_tag}") + if online: + remote_target_tag_exists = git_success(path, "ls-remote", "--exit-code", "--tags", "origin", f"refs/tags/{target_tag}", timeout=8) + + try: + versions = collect_versions(path) + except Exception as exc: # noqa: BLE001 - surfaced in the dashboard. + errors.append(str(exc)) + versions = VersionSnapshot() + + return RepositorySnapshot( + spec=spec, + absolute_path=str(path), + exists=True, + is_git=True, + has_head=has_head, + head=head, + branch=branch or None, + upstream=upstream or None, + ahead=ahead, + behind=behind, + dirty_entries=tuple(line for line in status_result.stdout.splitlines() if line.strip()), + versions=versions, + local_target_tag_exists=local_target_tag_exists, + remote_target_tag_exists=remote_target_tag_exists, + remote_checked=online, + errors=tuple(errors), + ) + + +def collect_versions(path: Path) -> VersionSnapshot: + return VersionSnapshot( + pyproject=read_pyproject_version(path), + package=read_json_version(path / "package.json"), + webui_package=read_json_version(path / "webui" / "package.json"), + manifests=read_manifest_versions(path), + ) + + +def read_pyproject_version(path: Path) -> str | None: + pyproject = path / "pyproject.toml" + if not pyproject.exists(): + return None + with pyproject.open("rb") as handle: + payload = tomllib.load(handle) + project = payload.get("project") + if isinstance(project, dict) and isinstance(project.get("version"), str): + return project["version"] + return None + + +def read_json_version(path: Path) -> str | None: + if not path.exists(): + return None + payload = json.loads(path.read_text(encoding="utf-8")) + version = payload.get("version") if isinstance(payload, dict) else None + return version if isinstance(version, str) else None + + +def read_manifest_versions(path: Path) -> tuple[str, ...]: + src = path / "src" + if not src.exists(): + return () + versions: list[str] = [] + for manifest in sorted(src.glob("**/backend/manifest.py")): + text = manifest.read_text(encoding="utf-8") + match = re.search(r'(?m)^\s*version\s*=\s*["\']([^"\']+)["\']', text) + if match is None: + match = re.search(r'(?m)^MODULE_VERSION\s*=\s*["\']([^"\']+)["\']', text) + if match is not None: + versions.append(match.group(1)) + return tuple(versions) + + +def git_text(path: Path, *args: str, timeout: int = 10) -> str: + result = git(path, *args, timeout=timeout) + return result.stdout.strip() if result.returncode == 0 else "" + + +def git_success(path: Path, *args: str, timeout: int = 10) -> bool: + return git(path, *args, timeout=timeout).returncode == 0 + + +def git(path: Path, *args: str, timeout: int = 10) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + ["git", *args], + cwd=path, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess(["git", *args], 124, exc.stdout or "", exc.stderr or "git command timed out") + + +def git_error(result: subprocess.CompletedProcess[str]) -> str: + return "\n".join(part for part in (result.stderr.strip(), result.stdout.strip()) if part) + + +def is_dubious_ownership_error(value: str) -> bool: + return "detected dubious ownership in repository" in value or "safe.directory" in value + + +def compact_git_error(value: str) -> str: + lines = [line.strip() for line in value.splitlines() if line.strip()] + return lines[0] if lines else "git command failed" + + +def safe_directory_command(path: Path) -> str: + return f"git config --global --add safe.directory {path}" diff --git a/tools/release/govoplan_release/http_fetch.py b/tools/release/govoplan_release/http_fetch.py new file mode 100644 index 0000000..f3ea512 --- /dev/null +++ b/tools/release/govoplan_release/http_fetch.py @@ -0,0 +1,44 @@ +"""Validated HTTP helpers for release tooling.""" + +from __future__ import annotations + +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Mapping + + +@dataclass(frozen=True, slots=True) +class HttpFetchResponse: + status: int + headers: dict[str, str] + body: bytes + + def text(self, encoding: str = "utf-8") -> str: + return self.body.decode(encoding) + + +def validate_http_url(value: str, *, label: str = "URL") -> str: + parsed = urllib.parse.urlparse(str(value).strip()) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError(f"{label} must be an absolute HTTP(S) URL.") + if parsed.username or parsed.password: + raise ValueError(f"{label} must not include embedded credentials.") + return urllib.parse.urlunparse(parsed) + + +def fetch_http( + url: str, + *, + timeout: float, + label: str = "URL", + method: str = "GET", + headers: Mapping[str, str] | None = None, +) -> HttpFetchResponse: + request = urllib.request.Request(validate_http_url(url, label=label), headers=dict(headers or {}), method=method) + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - URL is validated by validate_http_url. # nosec B310 # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected + return HttpFetchResponse( + status=int(getattr(response, "status", 0)), + headers=dict(response.headers.items()), + body=response.read(), + ) diff --git a/tools/release/govoplan_release/migrations.py b/tools/release/govoplan_release/migrations.py new file mode 100644 index 0000000..0c48529 --- /dev/null +++ b/tools/release/govoplan_release/migrations.py @@ -0,0 +1,53 @@ +"""Optional migration audit collection for the local release console.""" + +from __future__ import annotations + +import json +import subprocess +import sys + +from .model import MigrationTrackSnapshot +from .workspace import META_ROOT + + +def collect_migration_snapshots(*, timeout: int = 30) -> tuple[MigrationTrackSnapshot, ...]: + return tuple(collect_migration_snapshot(track, timeout=timeout) for track in ("release", "dev")) + + +def collect_migration_snapshot(track: str, *, timeout: int) -> MigrationTrackSnapshot: + script = META_ROOT / "tools" / "release" / "release-migration-audit.py" + try: + result = subprocess.run( + [sys.executable, str(script), "--track", track, "--json"], + cwd=META_ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return MigrationTrackSnapshot(track=track, ok=False, ran=False, error="migration audit timed out") + + if result.returncode != 0: + return MigrationTrackSnapshot( + track=track, + ok=False, + ran=True, + error=(result.stderr or result.stdout).strip() or f"migration audit exited with {result.returncode}", + ) + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + return MigrationTrackSnapshot(track=track, ok=False, ran=True, error=f"could not parse audit JSON: {exc}") + + graph_errors = tuple(str(item) for item in payload.get("graph_errors") or ()) + strict_errors = tuple(str(item) for item in payload.get("strict_errors") or ()) + return MigrationTrackSnapshot( + track=track, + ok=not graph_errors and not strict_errors, + ran=True, + graph_errors=graph_errors, + strict_errors=strict_errors, + ) diff --git a/tools/release/govoplan_release/model.py b/tools/release/govoplan_release/model.py new file mode 100644 index 0000000..793aad2 --- /dev/null +++ b/tools/release/govoplan_release/model.py @@ -0,0 +1,314 @@ +"""Data models for GovOPlaN release tooling. + +The models intentionally stay dataclass-based so both the CLI and the local web +console can use them without depending on a web framework or Pydantic. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, is_dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True, slots=True) +class RepositorySpec: + name: str + category: str + subtype: str + remote: str + path: str + + +@dataclass(frozen=True, slots=True) +class VersionSnapshot: + pyproject: str | None = None + package: str | None = None + webui_package: str | None = None + manifests: tuple[str, ...] = () + + @property + def primary(self) -> str | None: + return self.pyproject or self.package or self.webui_package or (self.manifests[0] if self.manifests else None) + + +@dataclass(frozen=True, slots=True) +class RepositorySnapshot: + spec: RepositorySpec + absolute_path: str + exists: bool + is_git: bool + has_head: bool = False + head: str | None = None + branch: str | None = None + upstream: str | None = None + ahead: int | None = None + behind: int | None = None + dirty_entries: tuple[str, ...] = () + versions: VersionSnapshot = VersionSnapshot() + local_target_tag_exists: bool | None = None + remote_target_tag_exists: bool | None = None + remote_checked: bool = False + safe_directory_required: bool = False + safe_directory_command: str | None = None + errors: tuple[str, ...] = () + + @property + def dirty(self) -> bool: + return bool(self.dirty_entries) + + +@dataclass(frozen=True, slots=True) +class CatalogSnapshot: + channel: str + website_path: str + catalog_path: str + catalog_exists: bool + keyring_path: str + keyring_exists: bool + catalog_version: str | None = None + core_release_version: str | None = None + module_count: int = 0 + signature_count: int = 0 + key_count: int = 0 + generated_at: str | None = None + expires_at: str | None = None + catalog_error: str | None = None + keyring_error: str | None = None + local_catalog_hash: str | None = None + local_keyring_hash: str | None = None + public_catalog_url: str | None = None + public_keyring_url: str | None = None + public_checked: bool = False + public_catalog_exists: bool | None = None + public_keyring_exists: bool | None = None + public_core_release_version: str | None = None + public_module_count: int | None = None + public_signature_count: int | None = None + public_key_count: int | None = None + public_catalog_hash: str | None = None + public_keyring_hash: str | None = None + public_catalog_error: str | None = None + public_keyring_error: str | None = None + catalog_matches_public: bool | None = None + keyring_matches_public: bool | None = None + + +@dataclass(frozen=True, slots=True) +class MigrationTrackSnapshot: + track: str + ok: bool + ran: bool + graph_errors: tuple[str, ...] = () + strict_errors: tuple[str, ...] = () + error: str | None = None + + +@dataclass(frozen=True, slots=True) +class DashboardSummary: + repository_count: int + missing_count: int + dirty_count: int + ahead_count: int + behind_count: int + no_head_count: int + error_count: int + safe_directory_count: int + local_target_tag_missing_count: int + status: str + + +@dataclass(frozen=True, slots=True) +class ReleaseDashboard: + generated_at: str + meta_root: str + workspace_root: str + target_version: str | None + target_tag: str | None + online: bool + include_migrations: bool + summary: DashboardSummary + repositories: tuple[RepositorySnapshot, ...] + catalog: CatalogSnapshot + migrations: tuple[MigrationTrackSnapshot, ...] = () + contracts: tuple[ModuleContractSnapshot, ...] = () + + +@dataclass(frozen=True, slots=True) +class PlanAction: + id: str + title: str + detail: str + command: str | None = None + cwd: str | None = None + mutating: bool = False + severity: str = "info" + repo: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReleasePlan: + generated_at: str + target_version: str | None + target_tag: str | None + status: str + actions: tuple[PlanAction, ...] + + +@dataclass(frozen=True, slots=True) +class InterfaceProviderSnapshot: + name: str + version: str + + +@dataclass(frozen=True, slots=True) +class InterfaceRequirementSnapshot: + name: str + version_min: str | None = None + version_max_exclusive: str | None = None + optional: bool = False + + +@dataclass(frozen=True, slots=True) +class ModuleContractSnapshot: + repo: str + module_id: str + module_version: str | None + manifest_path: str + provides_interfaces: tuple[InterfaceProviderSnapshot, ...] = () + requires_interfaces: tuple[InterfaceRequirementSnapshot, ...] = () + + +@dataclass(frozen=True, slots=True) +class ReleasePlanUnit: + repo: str + category: str + subtype: str + branch: str | None + current_version: str | None + target_version: str + target_tag: str + status: str + blockers: tuple[str, ...] = () + warnings: tuple[str, ...] = () + provides_interfaces: tuple[InterfaceProviderSnapshot, ...] = () + requires_interfaces: tuple[InterfaceRequirementSnapshot, ...] = () + + +@dataclass(frozen=True, slots=True) +class CompatibilityIssue: + severity: str + code: str + message: str + repo: str | None = None + module_id: str | None = None + interface: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReleasePlanStep: + id: str + title: str + detail: str + command: str | None = None + cwd: str | None = None + mutating: bool = False + repo: str | None = None + status: str = "planned" + + +@dataclass(frozen=True, slots=True) +class SelectiveReleasePlan: + generated_at: str + target_channel: str + status: str + units: tuple[ReleasePlanUnit, ...] + compatibility: tuple[CompatibilityIssue, ...] + dry_run_steps: tuple[ReleasePlanStep, ...] + notes: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class CatalogEntryChange: + repo: str + module_id: str | None + field: str + before: str | None + after: str | None + + +@dataclass(frozen=True, slots=True) +class SelectiveCatalogCandidate: + generated_at: str + channel: str + status: str + candidate_dir: str + catalog_path: str + keyring_path: str + summary_path: str | None + source_catalog: str + public_catalog_url: str + public_keyring_url: str + sequence: int + signature_count: int + key_count: int + candidate_catalog_hash: str + candidate_keyring_hash: str + published_catalog_hash: str | None + published_keyring_hash: str | None + candidate_matches_published_catalog: bool | None + candidate_matches_published_keyring: bool | None + validation_valid: bool + validation_error: str | None = None + validation_warnings: tuple[str, ...] = () + changes: tuple[CatalogEntryChange, ...] = () + notes: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class CatalogPublishStep: + id: str + title: str + detail: str + command: str | None = None + mutating: bool = False + status: str = "planned" + + +@dataclass(frozen=True, slots=True) +class CatalogPublishResult: + generated_at: str + channel: str + status: str + applied: bool + candidate_dir: str + candidate_catalog_path: str + candidate_keyring_path: str + web_root: str + target_catalog_path: str + target_keyring_path: str + branch: str | None + tag_name: str | None + validation_valid: bool + validation_error: str | None = None + validation_warnings: tuple[str, ...] = () + candidate_catalog_hash: str | None = None + candidate_keyring_hash: str | None = None + target_catalog_hash_before: str | None = None + target_keyring_hash_before: str | None = None + catalog_changed: bool = False + keyring_changed: bool = False + steps: tuple[CatalogPublishStep, ...] = () + notes: tuple[str, ...] = () + + +def to_jsonable(value: Any) -> Any: + if is_dataclass(value): + return {key: to_jsonable(item) for key, item in asdict(value).items()} + if isinstance(value, tuple | list): + return [to_jsonable(item) for item in value] + if isinstance(value, dict): + return {str(key): to_jsonable(item) for key, item in value.items()} + if isinstance(value, Path): + return str(value) + return value diff --git a/tools/release/govoplan_release/module_directory.py b/tools/release/govoplan_release/module_directory.py new file mode 100644 index 0000000..596e3fa --- /dev/null +++ b/tools/release/govoplan_release/module_directory.py @@ -0,0 +1,146 @@ +"""Generate browsable module-directory artifacts from a release catalog.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json +from pathlib import Path +import re +from typing import Any + +from .catalog import DEFAULT_PUBLIC_BASE_URL, canonical_hash +from .release_intelligence import compatibility_rows, module_rows, signature_rows + + +def write_module_directory( + *, + catalog_payload: dict[str, Any], + keyring_payload: dict[str, Any], + output_root: Path, + channel: str, + public_base_url: str = DEFAULT_PUBLIC_BASE_URL, +) -> tuple[Path, ...]: + payloads = module_directory_payloads( + catalog_payload=catalog_payload, + keyring_payload=keyring_payload, + channel=channel, + public_base_url=public_base_url, + ) + written: list[Path] = [] + for relative_path, payload in payloads: + path = output_root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + written.append(path) + return tuple(written) + + +def module_directory_payloads( + *, + catalog_payload: dict[str, Any], + keyring_payload: dict[str, Any], + channel: str, + public_base_url: str = DEFAULT_PUBLIC_BASE_URL, +) -> tuple[tuple[Path, dict[str, Any]], ...]: + generated_at = json_datetime(datetime.now(tz=UTC)) + modules = module_rows(catalog_payload) + compatibility = compatibility_rows(modules) + signatures = signature_rows(catalog_payload, keyring_payload) + base_url = public_base_url.rstrip("/") + catalog_url = f"{base_url}/catalogs/v1/channels/{channel}.json" + keyring_url = f"{base_url}/catalogs/v1/keyring.json" + catalog_hash = canonical_hash(catalog_payload) + catalog_sequence = catalog_payload.get("sequence") + + files: list[tuple[Path, dict[str, Any]]] = [] + module_index_entries: list[dict[str, Any]] = [] + for module in modules: + module_id = str(module.get("module_id") or module.get("repo") or "") + if not module_id: + continue + version = str(module.get("version") or "0.0.0") + module_slug = safe_path_part(module_id) + version_slug = safe_path_part(version) + module_base = f"{base_url}/catalogs/v1/modules/{module_slug}" + manifest_url = f"{module_base}/{version_slug}/manifest.json" + module_index_url = f"{module_base}/index.json" + module_compatibility = [row for row in compatibility if row.get("module_id") == module_id] + manifest = { + "manifest_version": "1", + "generated_at": generated_at, + "channel": channel, + "module": module, + "compatibility": module_compatibility, + "release": { + "catalog_url": catalog_url, + "keyring_url": keyring_url, + "catalog_hash": catalog_hash, + "catalog_sequence": catalog_sequence, + "signatures": signatures, + }, + } + files.append((Path("modules") / module_slug / version_slug / "manifest.json", manifest)) + files.append( + ( + Path("modules") / module_slug / "index.json", + { + "index_version": "1", + "generated_at": generated_at, + "channel": channel, + "module_id": module_id, + "name": module.get("name"), + "latest_version": version, + "latest_manifest_url": manifest_url, + "versions": [ + { + "version": version, + "manifest_url": manifest_url, + "python_tag": module.get("python_tag"), + "webui_tag": module.get("webui_tag"), + } + ], + "release": { + "catalog_url": catalog_url, + "keyring_url": keyring_url, + "catalog_hash": catalog_hash, + "catalog_sequence": catalog_sequence, + }, + }, + ) + ) + module_index_entries.append( + { + "module_id": module_id, + "name": module.get("name"), + "latest_version": version, + "index_url": module_index_url, + "latest_manifest_url": manifest_url, + "repo": module.get("repo"), + } + ) + + files.append( + ( + Path("modules") / "index.json", + { + "index_version": "1", + "generated_at": generated_at, + "channel": channel, + "catalog_url": catalog_url, + "keyring_url": keyring_url, + "catalog_hash": catalog_hash, + "catalog_sequence": catalog_sequence, + "module_count": len(module_index_entries), + "modules": sorted(module_index_entries, key=lambda item: str(item.get("module_id"))), + }, + ) + ) + return tuple(files) + + +def safe_path_part(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value.strip()) or "_" + + +def json_datetime(value: datetime) -> str: + return value.replace(microsecond=0).isoformat().replace("+00:00", "Z") diff --git a/tools/release/govoplan_release/planner.py b/tools/release/govoplan_release/planner.py new file mode 100644 index 0000000..078c8ef --- /dev/null +++ b/tools/release/govoplan_release/planner.py @@ -0,0 +1,176 @@ +"""Read-only release plan suggestions. + +This is deliberately not an executor yet. The first console slice should make +state and next actions explicit before any commit, tag, signing, or publish +operation becomes clickable. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +import shlex + +from .model import PlanAction, ReleaseDashboard, ReleasePlan + + +def build_release_plan(dashboard: ReleaseDashboard) -> ReleasePlan: + actions: list[PlanAction] = [] + target_version = dashboard.target_version + target_tag = dashboard.target_tag + + for repo in dashboard.repositories: + if not repo.exists: + actions.append( + PlanAction( + id=f"{repo.spec.name}:missing", + title=f"{repo.spec.name} is missing locally", + detail="Clone or bootstrap the repository before planning a release.", + command=f"tools/repo/bootstrap-repositories.py --repo {shlex.quote(repo.spec.name)}", + cwd=dashboard.meta_root, + mutating=True, + severity="blocker", + repo=repo.spec.name, + ) + ) + continue + if not repo.is_git: + actions.append( + PlanAction( + id=f"{repo.spec.name}:not-git", + title=f"{repo.spec.name} is not a Git repository", + detail=repo.absolute_path, + severity="blocker", + repo=repo.spec.name, + ) + ) + continue + if repo.safe_directory_required: + actions.append( + PlanAction( + id=f"{repo.spec.name}:safe-directory", + title=f"{repo.spec.name} requires Git safe.directory approval", + detail="Git refused to inspect this checkout until it is trusted for the current user.", + command=repo.safe_directory_command, + cwd=repo.absolute_path, + mutating=True, + severity="blocker", + repo=repo.spec.name, + ) + ) + continue + if not repo.has_head: + actions.append( + PlanAction( + id=f"{repo.spec.name}:initial-commit", + title=f"{repo.spec.name} needs an initial commit", + detail="The repository exists but has no HEAD yet, so it cannot be tagged or released.", + command="git add -A && git commit -m \"Initialize repository\" && git push -u origin HEAD", + cwd=repo.absolute_path, + mutating=True, + severity="blocker", + repo=repo.spec.name, + ) + ) + if repo.dirty: + actions.append( + PlanAction( + id=f"{repo.spec.name}:dirty", + title=f"{repo.spec.name} has uncommitted changes", + detail=f"{len(repo.dirty_entries)} changed path(s) need review before release.", + command="git status --short && git diff --stat", + cwd=repo.absolute_path, + severity="blocker", + repo=repo.spec.name, + ) + ) + if repo.behind: + actions.append( + PlanAction( + id=f"{repo.spec.name}:behind", + title=f"{repo.spec.name} is behind {repo.upstream}", + detail=f"Behind by {repo.behind} commit(s). Release from an up-to-date branch.", + command="git fetch --all --tags --prune", + cwd=repo.absolute_path, + mutating=True, + severity="blocker", + repo=repo.spec.name, + ) + ) + if repo.ahead: + actions.append( + PlanAction( + id=f"{repo.spec.name}:ahead", + title=f"{repo.spec.name} has unpushed commits", + detail=f"Ahead by {repo.ahead} commit(s). Tags and catalogs should point at pushed commits.", + command="git push", + cwd=repo.absolute_path, + mutating=True, + severity="warning", + repo=repo.spec.name, + ) + ) + if target_tag and repo.versions.primary == target_version and repo.local_target_tag_exists is False: + actions.append( + PlanAction( + id=f"{repo.spec.name}:tag-missing", + title=f"{repo.spec.name} has no local {target_tag} tag", + detail="The version file matches the target release, but the release tag is missing locally.", + command=f"git tag -a {shlex.quote(target_tag)} -m \"Release {shlex.quote(target_tag)}\"", + cwd=repo.absolute_path, + mutating=True, + severity="info", + repo=repo.spec.name, + ) + ) + + catalog = dashboard.catalog + if target_version and catalog.core_release_version != target_version: + actions.append( + PlanAction( + id="catalog:core-version", + title="Stable catalog does not point at the target core release", + detail=f"catalog core={catalog.core_release_version or '-'}, target={target_version}", + command=( + "KEY_DIR=\"$HOME/.config/govoplan/release-keys\" " + f"tools/release/publish-release-catalog.sh --version {shlex.quote(target_version)} " + "--catalog-signing-key \"release-key-1=$KEY_DIR/release-key-1.pem\" --build-web --dry-run" + ), + cwd=dashboard.meta_root, + severity="warning", + ) + ) + if catalog.catalog_exists and catalog.signature_count == 0: + actions.append( + PlanAction( + id="catalog:unsigned", + title="Stable catalog is unsigned", + detail="Release catalogs must be signed before operators can trust them.", + cwd=dashboard.meta_root, + severity="blocker", + ) + ) + if not catalog.keyring_exists or catalog.key_count == 0: + actions.append( + PlanAction( + id="catalog:keyring", + title="Catalog keyring is missing or empty", + detail="Generate or publish a public keyring before enforcing signed catalogs.", + command=( + "tools/release/generate-catalog-keypair.py --key-id release-key-1 " + "--private-key \"$HOME/.config/govoplan/release-keys/release-key-1.pem\" " + "--keyring \"$HOME/.config/govoplan/release-keys/keyring.json\"" + ), + cwd=dashboard.meta_root, + mutating=True, + severity="warning", + ) + ) + + status = "blocked" if any(action.severity == "blocker" for action in actions) else "attention" if actions else "ready" + return ReleasePlan( + generated_at=datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + target_version=target_version, + target_tag=target_tag, + status=status, + actions=tuple(actions), + ) diff --git a/tools/release/govoplan_release/publisher.py b/tools/release/govoplan_release/publisher.py new file mode 100644 index 0000000..850d848 --- /dev/null +++ b/tools/release/govoplan_release/publisher.py @@ -0,0 +1,409 @@ +"""Publish reviewed release catalog candidates into the website repository.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json +from pathlib import Path +import shlex +import shutil +import subprocess + +from govoplan_core.core.module_package_catalog import validate_module_package_catalog + +from .catalog import canonical_hash +from .model import CatalogPublishResult, CatalogPublishStep +from .selective_catalog import trusted_keys_from_keyring +from .workspace import DEFAULT_WORKSPACE_ROOT, resolve_workspace_root, website_root + + +def publish_catalog_candidate( + *, + candidate_dir: Path | str, + workspace_root: Path | str | None = None, + web_root: Path | str | None = None, + channel: str = "stable", + apply: bool = False, # noqa: A002 - CLI uses --apply. + build_web: bool = False, + commit: bool = False, + tag: bool = False, + push: bool = False, + remote: str = "origin", + branch: str | None = None, + npm: str = "npm", + tag_name: str | None = None, + allow_dirty_website: bool = False, +) -> CatalogPublishResult: + workspace = resolve_workspace_root(workspace_root or DEFAULT_WORKSPACE_ROOT) + candidate_root = Path(candidate_dir).expanduser() + if not candidate_root.is_absolute(): + candidate_root = Path.cwd() / candidate_root + resolved_web_root = Path(web_root).expanduser().resolve() if web_root is not None else website_root(workspace) + candidate_catalog = candidate_root / "channels" / f"{channel}.json" + candidate_keyring = candidate_root / "keyring.json" + candidate_modules = candidate_root / "modules" + target_catalog = resolved_web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json" + target_keyring = resolved_web_root / "public" / "catalogs" / "v1" / "keyring.json" + target_modules = resolved_web_root / "public" / "catalogs" / "v1" / "modules" + + steps: list[CatalogPublishStep] = [] + notes: list[str] = [] + blockers: list[str] = [] + + if not candidate_catalog.exists(): + blockers.append(f"candidate catalog is missing: {candidate_catalog}") + if not candidate_keyring.exists(): + blockers.append(f"candidate keyring is missing: {candidate_keyring}") + if not resolved_web_root.exists(): + blockers.append(f"website root is missing: {resolved_web_root}") + if push: + commit = True + if tag: + commit = True + + candidate_payload = read_json(candidate_catalog) if candidate_catalog.exists() else None + keyring_payload = read_json(candidate_keyring) if candidate_keyring.exists() else None + candidate_catalog_hash = canonical_hash(candidate_payload) if candidate_payload is not None else None + candidate_keyring_hash = canonical_hash(keyring_payload) if keyring_payload is not None else None + target_catalog_hash_before = file_json_hash(target_catalog) + target_keyring_hash_before = file_json_hash(target_keyring) + validation_valid = False + validation_error: str | None = None + validation_warnings: tuple[str, ...] = () + + if candidate_payload is not None and keyring_payload is not None: + validation = validate_module_package_catalog( + candidate_catalog, + require_trusted=True, + approved_channels=(channel,), + trusted_keys=trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {}), + ) + validation_valid = validation.get("valid") is True + validation_error = str(validation["error"]) if validation.get("error") else None + validation_warnings = tuple(str(item) for item in validation.get("warnings") or ()) + if not validation_valid: + blockers.append(validation_error or "candidate catalog validation failed") + + catalog_changed = candidate_catalog_hash != target_catalog_hash_before + keyring_changed = candidate_keyring_hash != target_keyring_hash_before + if not catalog_changed and not keyring_changed: + notes.append("Candidate catalog and keyring already match the website repository.") + + if apply or commit or tag or push: + if not (resolved_web_root / ".git").exists(): + blockers.append(f"website root is not a Git repository: {resolved_web_root}") + if not allow_dirty_website and website_dirty(resolved_web_root): + blockers.append("website repository has uncommitted changes") + + effective_branch = branch or git_text(resolved_web_root, "branch", "--show-current") or None + effective_tag_name = tag_name or default_tag_name(candidate_payload, channel=channel) + if (tag or push) and not effective_tag_name: + blockers.append("could not infer catalog tag name; pass --tag-name") + if (commit or push) and not effective_branch: + blockers.append("could not infer website branch; pass --branch") + if tag and effective_tag_name and git_success(resolved_web_root, "rev-parse", "--quiet", "--verify", f"refs/tags/{effective_tag_name}"): + blockers.append(f"website tag already exists: {effective_tag_name}") + + steps.append( + CatalogPublishStep( + id="validate", + title="Validate candidate catalog", + detail="Validate signature, approved channel, freshness, interface closure, and keyring trust.", + status="done" if validation_valid else "blocked", + ) + ) + steps.append( + CatalogPublishStep( + id="copy", + title="Copy candidate catalog, keyring, and module directory", + detail=f"{candidate_root} -> {resolved_web_root / 'public' / 'catalogs' / 'v1'}", + mutating=True, + status="planned" if not apply else "blocked" if blockers else "done", + ) + ) + if not candidate_modules.exists(): + notes.append("Candidate has no module-directory tree; only catalog and keyring will be copied.") + if build_web: + steps.append( + CatalogPublishStep( + id="build-web", + title="Build website", + detail="Run the website build after copying catalog files.", + command=shlex.join([npm, "--prefix", str(resolved_web_root), "run", "build"]), + mutating=True, + status="planned" if not apply else "blocked" if blockers else "pending", + ) + ) + if commit: + steps.append( + CatalogPublishStep( + id="commit", + title="Commit website catalog update", + detail="Commit generated catalog/keyring files in the website repository.", + command=shlex.join(["git", "-C", str(resolved_web_root), "commit", "-m", commit_message(candidate_payload, channel=channel)]), + mutating=True, + status="planned" if not apply else "blocked" if blockers else "pending", + ) + ) + if tag: + steps.append( + CatalogPublishStep( + id="tag", + title=f"Tag website catalog publication {effective_tag_name}", + detail="Create an annotated website publication tag.", + command=shlex.join(["git", "-C", str(resolved_web_root), "tag", "-a", effective_tag_name, "-m", commit_message(candidate_payload, channel=channel)]), + mutating=True, + status="planned" if not apply else "blocked" if blockers else "pending", + ) + ) + if push: + steps.append( + CatalogPublishStep( + id="push", + title="Push website catalog publication", + detail="Push website branch and tag if a tag was requested.", + command=push_command(resolved_web_root, remote=remote, branch=effective_branch, tag_name=effective_tag_name if tag else None), + mutating=True, + status="planned" if not apply else "blocked" if blockers else "pending", + ) + ) + + if blockers: + return result( + status="blocked", + applied=False, + candidate_root=candidate_root, + candidate_catalog=candidate_catalog, + candidate_keyring=candidate_keyring, + resolved_web_root=resolved_web_root, + target_catalog=target_catalog, + target_keyring=target_keyring, + channel=channel, + branch=effective_branch, + tag_name=effective_tag_name, + validation_valid=validation_valid, + validation_error=validation_error, + validation_warnings=validation_warnings, + candidate_catalog_hash=candidate_catalog_hash, + candidate_keyring_hash=candidate_keyring_hash, + target_catalog_hash_before=target_catalog_hash_before, + target_keyring_hash_before=target_keyring_hash_before, + catalog_changed=catalog_changed, + keyring_changed=keyring_changed, + steps=tuple(steps), + notes=tuple([*notes, *blockers]), + ) + + if not apply: + notes.append("Dry run only. Pass --apply to copy files; pass --commit/--tag/--push for Git publication steps.") + return result( + status="ready", + applied=False, + candidate_root=candidate_root, + candidate_catalog=candidate_catalog, + candidate_keyring=candidate_keyring, + resolved_web_root=resolved_web_root, + target_catalog=target_catalog, + target_keyring=target_keyring, + channel=channel, + branch=effective_branch, + tag_name=effective_tag_name, + validation_valid=validation_valid, + validation_error=validation_error, + validation_warnings=validation_warnings, + candidate_catalog_hash=candidate_catalog_hash, + candidate_keyring_hash=candidate_keyring_hash, + target_catalog_hash_before=target_catalog_hash_before, + target_keyring_hash_before=target_keyring_hash_before, + catalog_changed=catalog_changed, + keyring_changed=keyring_changed, + steps=tuple(steps), + notes=tuple(notes), + ) + + target_catalog.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(candidate_catalog, target_catalog) + shutil.copy2(candidate_keyring, target_keyring) + if candidate_modules.exists(): + if target_modules.exists(): + shutil.rmtree(target_modules) + shutil.copytree(candidate_modules, target_modules) + + completed_steps = [replace_status(step, "done") if step.id == "copy" else step for step in steps] + if build_web: + run_checked([npm, "--prefix", str(resolved_web_root), "run", "build"], cwd=resolved_web_root) + completed_steps = [replace_status(step, "done") if step.id == "build-web" else step for step in completed_steps] + if commit: + add_paths = ["git", "-C", str(resolved_web_root), "add", str(target_catalog), str(target_keyring)] + if target_modules.exists(): + add_paths.append(str(target_modules)) + run_checked(add_paths, cwd=resolved_web_root) + if git_success(resolved_web_root, "diff", "--cached", "--quiet"): + notes.append("No website catalog changes were staged for commit.") + completed_steps = [replace_status(step, "skipped") if step.id == "commit" else step for step in completed_steps] + else: + run_checked(["git", "-C", str(resolved_web_root), "commit", "-m", commit_message(candidate_payload, channel=channel)], cwd=resolved_web_root) + completed_steps = [replace_status(step, "done") if step.id == "commit" else step for step in completed_steps] + if tag and effective_tag_name: + run_checked(["git", "-C", str(resolved_web_root), "tag", "-a", effective_tag_name, "-m", commit_message(candidate_payload, channel=channel)], cwd=resolved_web_root) + completed_steps = [replace_status(step, "done") if step.id == "tag" else step for step in completed_steps] + if push and effective_branch: + push_args = ["git", "-C", str(resolved_web_root), "push"] + if tag and effective_tag_name: + push_args.extend(["--atomic", remote, f"HEAD:refs/heads/{effective_branch}", f"refs/tags/{effective_tag_name}"]) + else: + push_args.extend([remote, f"HEAD:refs/heads/{effective_branch}"]) + run_checked(push_args, cwd=resolved_web_root) + completed_steps = [replace_status(step, "done") if step.id == "push" else step for step in completed_steps] + + return result( + status="published" if push else "applied", + applied=True, + candidate_root=candidate_root, + candidate_catalog=candidate_catalog, + candidate_keyring=candidate_keyring, + resolved_web_root=resolved_web_root, + target_catalog=target_catalog, + target_keyring=target_keyring, + channel=channel, + branch=effective_branch, + tag_name=effective_tag_name, + validation_valid=validation_valid, + validation_error=validation_error, + validation_warnings=validation_warnings, + candidate_catalog_hash=candidate_catalog_hash, + candidate_keyring_hash=candidate_keyring_hash, + target_catalog_hash_before=target_catalog_hash_before, + target_keyring_hash_before=target_keyring_hash_before, + catalog_changed=catalog_changed, + keyring_changed=keyring_changed, + steps=tuple(completed_steps), + notes=tuple(notes), + ) + + +def result( + *, + status: str, + applied: bool, + candidate_root: Path, + candidate_catalog: Path, + candidate_keyring: Path, + resolved_web_root: Path, + target_catalog: Path, + target_keyring: Path, + channel: str, + branch: str | None, + tag_name: str | None, + validation_valid: bool, + validation_error: str | None, + validation_warnings: tuple[str, ...], + candidate_catalog_hash: str | None, + candidate_keyring_hash: str | None, + target_catalog_hash_before: str | None, + target_keyring_hash_before: str | None, + catalog_changed: bool, + keyring_changed: bool, + steps: tuple[CatalogPublishStep, ...], + notes: tuple[str, ...], +) -> CatalogPublishResult: + return CatalogPublishResult( + generated_at=datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + channel=channel, + status=status, + applied=applied, + candidate_dir=str(candidate_root), + candidate_catalog_path=str(candidate_catalog), + candidate_keyring_path=str(candidate_keyring), + web_root=str(resolved_web_root), + target_catalog_path=str(target_catalog), + target_keyring_path=str(target_keyring), + branch=branch, + tag_name=tag_name, + validation_valid=validation_valid, + validation_error=validation_error, + validation_warnings=validation_warnings, + candidate_catalog_hash=candidate_catalog_hash, + candidate_keyring_hash=candidate_keyring_hash, + target_catalog_hash_before=target_catalog_hash_before, + target_keyring_hash_before=target_keyring_hash_before, + catalog_changed=catalog_changed, + keyring_changed=keyring_changed, + steps=steps, + notes=notes, + ) + + +def read_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def file_json_hash(path: Path) -> str | None: + if not path.exists(): + return None + return canonical_hash(read_json(path)) + + +def website_dirty(path: Path) -> bool: + if not (path / ".git").exists(): + return False + return bool(git_text(path, "status", "--porcelain")) + + +def default_tag_name(payload: object, *, channel: str) -> str | None: + if not isinstance(payload, dict): + return None + sequence = payload.get("sequence") + release = payload.get("release") + selected_units = release.get("selected_units") if isinstance(release, dict) else None + if isinstance(selected_units, list) and len(selected_units) == 1 and isinstance(selected_units[0], dict): + repo = str(selected_units[0].get("repo") or "catalog") + version = str(selected_units[0].get("version") or "").removeprefix("v") + if version: + return f"catalog-{channel}-{repo}-v{version}" + return f"catalog-{channel}-{sequence}" if sequence is not None else None + + +def commit_message(payload: object, *, channel: str) -> str: + if isinstance(payload, dict): + release = payload.get("release") + selected_units = release.get("selected_units") if isinstance(release, dict) else None + if isinstance(selected_units, list) and selected_units: + units = ", ".join( + f"{item.get('repo')} v{str(item.get('version') or '').removeprefix('v')}" + for item in selected_units + if isinstance(item, dict) + ) + if units: + return f"Publish {channel} catalog for {units}" + return f"Publish {channel} catalog" + + +def push_command(web_root: Path, *, remote: str, branch: str | None, tag_name: str | None) -> str: + if tag_name: + return shlex.join(["git", "-C", str(web_root), "push", "--atomic", remote, f"HEAD:refs/heads/{branch or ''}", f"refs/tags/{tag_name}"]) + return shlex.join(["git", "-C", str(web_root), "push", remote, f"HEAD:refs/heads/{branch or ''}"]) + + +def replace_status(step: CatalogPublishStep, status: str) -> CatalogPublishStep: + return CatalogPublishStep( + id=step.id, + title=step.title, + detail=step.detail, + command=step.command, + mutating=step.mutating, + status=status, + ) + + +def git_text(path: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=path, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return result.stdout.strip() if result.returncode == 0 else "" + + +def git_success(path: Path, *args: str) -> bool: + return subprocess.run(["git", *args], cwd=path, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0 + + +def run_checked(args: list[str], *, cwd: Path) -> None: + subprocess.run(args, cwd=cwd, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/tools/release/govoplan_release/release_intelligence.py b/tools/release/govoplan_release/release_intelligence.py new file mode 100644 index 0000000..473c4a1 --- /dev/null +++ b/tools/release/govoplan_release/release_intelligence.py @@ -0,0 +1,434 @@ +"""Release catalog intelligence for the local release cockpit.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json +from pathlib import Path +import re + +from .catalog import DEFAULT_PUBLIC_BASE_URL, canonical_hash, fetch_json +from .git_state import collect_repository_snapshot +from .workspace import load_repository_specs, resolve_workspace_root, website_root + + +def build_release_intelligence( + *, + workspace_root: Path | str | None = None, + channel: str = "stable", + public_base_url: str = DEFAULT_PUBLIC_BASE_URL, + prefer_public: bool = True, +) -> dict[str, object]: + workspace = resolve_workspace_root(workspace_root) + web_root = website_root(workspace) + local_catalog_path = web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json" + local_keyring_path = web_root / "public" / "catalogs" / "v1" / "keyring.json" + public_catalog_url = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json" + public_keyring_url = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json" + + local_catalog = read_json_file(local_catalog_path) + local_keyring = read_json_file(local_keyring_path) + public_catalog = fetch_json(public_catalog_url) if prefer_public else {"ok": False, "error": "public check disabled"} + public_keyring = fetch_json(public_keyring_url) if prefer_public else {"ok": False, "error": "public check disabled"} + + catalog_source = "public" if prefer_public and public_catalog.get("ok") else "local" + keyring_source = "public" if prefer_public and public_keyring.get("ok") else "local" + catalog_payload = public_catalog["payload"] if catalog_source == "public" else local_catalog.get("payload") + keyring_payload = public_keyring["payload"] if keyring_source == "public" else local_keyring.get("payload") + + modules = module_rows(catalog_payload) + compatibility = compatibility_rows(modules) + signatures = signature_rows(catalog_payload, keyring_payload) + local_versions = repository_versions(workspace) + for module in modules: + repo = module.get("repo") + module["local_version"] = local_versions.get(repo) if isinstance(repo, str) else None + module["local_matches_catalog"] = module.get("version") == module.get("local_version") if module.get("local_version") else None + + local_catalog_hash = canonical_hash(local_catalog["payload"]) if local_catalog.get("ok") else None + public_catalog_hash = canonical_hash(public_catalog["payload"]) if public_catalog.get("ok") else None + local_keyring_hash = canonical_hash(local_keyring["payload"]) if local_keyring.get("ok") else None + public_keyring_hash = canonical_hash(public_keyring["payload"]) if public_keyring.get("ok") else None + issues = health_issues( + catalog_payload=catalog_payload, + keyring_payload=keyring_payload, + local_catalog_hash=local_catalog_hash, + public_catalog_hash=public_catalog_hash, + local_keyring_hash=local_keyring_hash, + public_keyring_hash=public_keyring_hash, + signatures=signatures, + compatibility=compatibility, + ) + return { + "generated_at": json_datetime(datetime.now(tz=UTC)), + "channel": channel, + "catalog_source": catalog_source, + "keyring_source": keyring_source, + "catalog": catalog_summary( + catalog_payload, + source=catalog_source, + local_path=str(local_catalog_path), + public_url=public_catalog_url, + local_ok=bool(local_catalog.get("ok")), + public_ok=bool(public_catalog.get("ok")), + local_hash=local_catalog_hash, + public_hash=public_catalog_hash, + public_error=str(public_catalog.get("error")) if public_catalog.get("error") else None, + ), + "keyring": keyring_summary( + keyring_payload, + source=keyring_source, + local_path=str(local_keyring_path), + public_url=public_keyring_url, + local_ok=bool(local_keyring.get("ok")), + public_ok=bool(public_keyring.get("ok")), + local_hash=local_keyring_hash, + public_hash=public_keyring_hash, + public_error=str(public_keyring.get("error")) if public_keyring.get("error") else None, + ), + "modules": modules, + "signatures": signatures, + "compatibility": compatibility, + "issues": issues, + "summary": { + "status": status_from_issues(issues), + "published_modules": len(modules), + "signed": bool(signatures), + "trusted_signatures": sum(1 for item in signatures if item.get("trusted") is True), + "compatibility_blockers": sum(1 for item in compatibility if item.get("severity") == "blocker"), + "compatibility_warnings": sum(1 for item in compatibility if item.get("severity") == "warning"), + "catalog_matches_public": compare_hashes(local_catalog_hash, public_catalog_hash), + "keyring_matches_public": compare_hashes(local_keyring_hash, public_keyring_hash), + }, + } + + +def read_json_file(path: Path) -> dict[str, object]: + if not path.exists(): + return {"ok": False, "error": "missing"} + try: + return {"ok": True, "payload": json.loads(path.read_text(encoding="utf-8"))} + except (OSError, json.JSONDecodeError) as exc: + return {"ok": False, "error": str(exc)} + + +def catalog_summary( + payload: object, + *, + source: str, + local_path: str, + public_url: str, + local_ok: bool, + public_ok: bool, + local_hash: str | None, + public_hash: str | None, + public_error: str | None, +) -> dict[str, object]: + catalog = payload if isinstance(payload, dict) else {} + core = catalog.get("core_release") if isinstance(catalog.get("core_release"), dict) else {} + modules = catalog.get("modules") if isinstance(catalog.get("modules"), list) else [] + signatures = catalog.get("signatures") if isinstance(catalog.get("signatures"), list) else [] + return { + "source": source, + "local_path": local_path, + "public_url": public_url, + "local_ok": local_ok, + "public_ok": public_ok, + "public_error": public_error, + "local_hash": local_hash, + "public_hash": public_hash, + "matches_public": compare_hashes(local_hash, public_hash), + "catalog_version": catalog.get("catalog_version"), + "sequence": catalog.get("sequence"), + "generated_at": catalog.get("generated_at"), + "expires_at": catalog.get("expires_at"), + "core_version": core.get("version") if isinstance(core, dict) else None, + "module_count": len(modules), + "signature_count": len(signatures), + } + + +def keyring_summary( + payload: object, + *, + source: str, + local_path: str, + public_url: str, + local_ok: bool, + public_ok: bool, + local_hash: str | None, + public_hash: str | None, + public_error: str | None, +) -> dict[str, object]: + keyring = payload if isinstance(payload, dict) else {} + keys = keyring.get("keys") if isinstance(keyring.get("keys"), list) else [] + return { + "source": source, + "local_path": local_path, + "public_url": public_url, + "local_ok": local_ok, + "public_ok": public_ok, + "public_error": public_error, + "local_hash": local_hash, + "public_hash": public_hash, + "matches_public": compare_hashes(local_hash, public_hash), + "generated_at": keyring.get("generated_at"), + "key_count": len(keys), + "active_key_count": sum(1 for item in keys if isinstance(item, dict) and item.get("status") == "active"), + } + + +def module_rows(payload: object) -> list[dict[str, object]]: + catalog = payload if isinstance(payload, dict) else {} + raw_modules = catalog.get("modules") if isinstance(catalog.get("modules"), list) else [] + modules: list[dict[str, object]] = [] + for item in raw_modules: + if not isinstance(item, dict): + continue + repo = module_repo(item) + modules.append( + { + "module_id": string(item.get("module_id")), + "name": string(item.get("name")), + "version": string(item.get("version")), + "repo": repo, + "python_ref": string(item.get("python_ref")), + "python_tag": ref_tag(string(item.get("python_ref"))), + "webui_ref": string(item.get("webui_ref")), + "webui_tag": ref_tag(string(item.get("webui_ref"))), + "provides_interfaces": interface_list(item.get("provides_interfaces")), + "requires_interfaces": requirement_list(item.get("requires_interfaces")), + "migration_safety": string(item.get("migration_safety")), + "tags": tuple(str(value) for value in item.get("tags", ()) if isinstance(value, str)) if isinstance(item.get("tags"), list) else (), + } + ) + return sorted(modules, key=lambda row: str(row.get("module_id") or row.get("repo") or "")) + + +def module_repo(entry: dict[str, object]) -> str | None: + for field in ("python_ref", "webui_ref"): + value = entry.get(field) + if isinstance(value, str): + match = re.search(r"/([^/@#]+)[.]git(?:[@#]|$)", value) + if match: + return match.group(1) + package = entry.get("python_package") + if isinstance(package, str) and package.startswith("govoplan-"): + return package.split("[", 1)[0] + return None + + +def ref_tag(value: str | None) -> str | None: + if not value: + return None + if "#" in value: + return value.rsplit("#", 1)[-1] + if "@" in value: + return value.rsplit("@", 1)[-1] + return None + + +def interface_list(value: object) -> tuple[dict[str, str], ...]: + if not isinstance(value, list): + return () + result: list[dict[str, str]] = [] + for item in value: + if not isinstance(item, dict): + continue + name = string(item.get("name")) + version = string(item.get("version")) + if name and version: + result.append({"name": name, "version": version}) + return tuple(result) + + +def requirement_list(value: object) -> tuple[dict[str, object], ...]: + if not isinstance(value, list): + return () + result: list[dict[str, object]] = [] + for item in value: + if not isinstance(item, dict): + continue + name = string(item.get("name")) + if not name: + continue + result.append( + { + "name": name, + "version_min": string(item.get("version_min")), + "version_max_exclusive": string(item.get("version_max_exclusive")), + "optional": bool(item.get("optional")), + } + ) + return tuple(result) + + +def compatibility_rows(modules: list[dict[str, object]]) -> list[dict[str, object]]: + providers: dict[str, list[dict[str, object]]] = {} + for module in modules: + for provided in module.get("provides_interfaces", ()): + if isinstance(provided, dict): + providers.setdefault(str(provided.get("name")), []).append( + { + "module_id": module.get("module_id"), + "repo": module.get("repo"), + "version": provided.get("version"), + } + ) + + rows: list[dict[str, object]] = [] + for module in modules: + for requirement in module.get("requires_interfaces", ()): + if not isinstance(requirement, dict): + continue + name = string(requirement.get("name")) + if not name: + continue + optional = bool(requirement.get("optional")) + available = providers.get(name, []) + satisfied = [provider for provider in available if version_satisfies(string(provider.get("version")), requirement)] + if satisfied: + severity = "ok" + status = "satisfied" + elif available: + severity = "warning" if optional else "blocker" + status = "version-mismatch" + else: + severity = "info" if optional else "blocker" + status = "optional-missing" if optional else "missing" + rows.append( + { + "module_id": module.get("module_id"), + "repo": module.get("repo"), + "interface": name, + "requirement": format_requirement(requirement), + "optional": optional, + "status": status, + "severity": severity, + "providers": available, + "satisfied_by": satisfied, + } + ) + return sorted(rows, key=lambda row: (str(row.get("severity")), str(row.get("module_id")), str(row.get("interface")))) + + +def signature_rows(catalog_payload: object, keyring_payload: object) -> list[dict[str, object]]: + catalog = catalog_payload if isinstance(catalog_payload, dict) else {} + keyring = keyring_payload if isinstance(keyring_payload, dict) else {} + signatures = catalog.get("signatures") if isinstance(catalog.get("signatures"), list) else [] + keys = keyring.get("keys") if isinstance(keyring.get("keys"), list) else [] + trusted = { + string(item.get("key_id")): item + for item in keys + if isinstance(item, dict) and string(item.get("key_id")) + } + rows: list[dict[str, object]] = [] + for item in signatures: + if not isinstance(item, dict): + continue + key_id = string(item.get("key_id")) + key = trusted.get(key_id) + rows.append( + { + "key_id": key_id, + "algorithm": string(item.get("algorithm")), + "trusted": key is not None and key.get("status") == "active", + "key_status": string(key.get("status")) if isinstance(key, dict) else None, + } + ) + return rows + + +def repository_versions(workspace: Path) -> dict[str | None, str | None]: + versions: dict[str | None, str | None] = {} + for spec in load_repository_specs(include_website=False): + snapshot = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + versions[spec.name] = snapshot.versions.primary + return versions + + +def health_issues( + *, + catalog_payload: object, + keyring_payload: object, + local_catalog_hash: str | None, + public_catalog_hash: str | None, + local_keyring_hash: str | None, + public_keyring_hash: str | None, + signatures: list[dict[str, object]], + compatibility: list[dict[str, object]], +) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + if not isinstance(catalog_payload, dict): + issues.append({"severity": "blocker", "code": "catalog_missing", "message": "No usable catalog is available."}) + if not isinstance(keyring_payload, dict): + issues.append({"severity": "blocker", "code": "keyring_missing", "message": "No usable keyring is available."}) + if not signatures: + issues.append({"severity": "blocker", "code": "catalog_unsigned", "message": "The selected catalog has no signatures."}) + elif not any(item.get("trusted") is True for item in signatures): + issues.append({"severity": "blocker", "code": "signature_untrusted", "message": "No catalog signature matches an active keyring key."}) + if compare_hashes(local_catalog_hash, public_catalog_hash) is False: + issues.append({"severity": "warning", "code": "catalog_drift", "message": "Local and published catalogs differ."}) + if compare_hashes(local_keyring_hash, public_keyring_hash) is False: + issues.append({"severity": "warning", "code": "keyring_drift", "message": "Local and published keyrings differ."}) + blockers = [item for item in compatibility if item.get("severity") == "blocker"] + warnings = [item for item in compatibility if item.get("severity") == "warning"] + if blockers: + issues.append({"severity": "blocker", "code": "compatibility_blockers", "message": f"{len(blockers)} compatibility blocker(s) found."}) + if warnings: + issues.append({"severity": "warning", "code": "compatibility_warnings", "message": f"{len(warnings)} compatibility warning(s) found."}) + return issues + + +def version_satisfies(version: str | None, requirement: dict[str, object]) -> bool: + parsed = parse_version(version) + if parsed is None: + return False + minimum = parse_version(string(requirement.get("version_min"))) + maximum = parse_version(string(requirement.get("version_max_exclusive"))) + if minimum is not None and parsed < minimum: + return False + if maximum is not None and parsed >= maximum: + return False + return True + + +def parse_version(value: str | None) -> tuple[int, ...] | None: + if not value: + return None + core = value.removeprefix("v").split("-", 1)[0] + parts = core.split(".") + if not parts or any(not part.isdigit() for part in parts): + return None + return tuple(int(part) for part in parts) + + +def format_requirement(requirement: dict[str, object]) -> str: + parts: list[str] = [] + if requirement.get("version_min"): + parts.append(f">={requirement['version_min']}") + if requirement.get("version_max_exclusive"): + parts.append(f"<{requirement['version_max_exclusive']}") + return ", ".join(parts) if parts else "any" + + +def status_from_issues(issues: list[dict[str, str]]) -> str: + if any(item.get("severity") == "blocker" for item in issues): + return "blocked" + if any(item.get("severity") == "warning" for item in issues): + return "attention" + return "ready" + + +def compare_hashes(local_hash: str | None, public_hash: str | None) -> bool | None: + if local_hash is None or public_hash is None: + return None + return local_hash == public_hash + + +def string(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def json_datetime(value: datetime) -> str: + return value.replace(microsecond=0).isoformat().replace("+00:00", "Z") diff --git a/tools/release/govoplan_release/repository_prepare.py b/tools/release/govoplan_release/repository_prepare.py new file mode 100644 index 0000000..6ef50af --- /dev/null +++ b/tools/release/govoplan_release/repository_prepare.py @@ -0,0 +1,137 @@ +"""Prepare selected repositories by committing reviewed dirty worktrees.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess + +from .git_state import collect_repository_snapshot +from .repository_push import command_text, compact_output +from .workspace import load_repository_specs, resolve_repo_path, resolve_workspace_root + + +def prepare_repositories( + *, + repos: tuple[str, ...], + repo_versions: dict[str, str] | None = None, + message: str | None = None, + workspace_root: Path | str | None = None, + apply: bool = False, # noqa: A002 - mirrors API field. +) -> dict[str, object]: + workspace = resolve_workspace_root(workspace_root) + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + selected = tuple(dict.fromkeys(repos)) + versions = repo_versions or {} + results: list[dict[str, object]] = [] + + for repo in selected: + spec = specs.get(repo) + if spec is None: + results.append({"repo": repo, "status": "blocked", "detail": "repository is not listed in repositories.json"}) + continue + snapshot = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + commit_message = resolve_message(repo=repo, version=versions.get(repo), message=message) + command = f"{command_text(('git', 'add', '-A'))} && {command_text(('git', 'commit', '-m', commit_message))}" + row: dict[str, object] = { + "repo": repo, + "cwd": snapshot.absolute_path, + "branch": snapshot.branch, + "upstream": snapshot.upstream, + "head": snapshot.head, + "ahead": int(snapshot.ahead or 0), + "behind": int(snapshot.behind or 0), + "dirty_count": len(snapshot.dirty_entries), + "dirty_entries": snapshot.dirty_entries, + "message": commit_message, + "command": command, + } + if not snapshot.exists or not snapshot.is_git: + results.append({**row, "status": "blocked", "detail": "repository is missing or is not a Git repository"}) + continue + if snapshot.safe_directory_required: + results.append({**row, "status": "blocked", "detail": "Git safe.directory approval is required"}) + continue + if not snapshot.has_head: + results.append({**row, "status": "blocked", "detail": "repository has no HEAD; initialize it outside the release prepare step"}) + continue + if not snapshot.branch: + results.append({**row, "status": "blocked", "detail": "repository is not on a named branch"}) + continue + if snapshot.behind: + results.append({**row, "status": "blocked", "detail": f"repository is behind {snapshot.upstream}; sync/update before committing"}) + continue + if not snapshot.dirty_entries: + results.append({**row, "status": "noop", "detail": "worktree is clean; no prepare commit needed"}) + continue + if not apply: + results.append({**row, "status": "planned", "detail": f"{len(snapshot.dirty_entries)} dirty/untracked path(s) will be committed"}) + continue + + path = resolve_repo_path(spec, workspace) + add_result = run(("git", "add", "-A"), cwd=path) + if add_result.returncode != 0: + results.append( + { + **row, + "status": "failed", + "returncode": add_result.returncode, + "detail": "git add failed", + "stdout": compact_output(add_result.stdout), + "stderr": compact_output(add_result.stderr), + } + ) + continue + if run(("git", "diff", "--cached", "--quiet"), cwd=path).returncode == 0: + results.append({**row, "status": "noop", "detail": "nothing staged after git add -A"}) + continue + commit_result = run(("git", "commit", "-m", commit_message), cwd=path) + if commit_result.returncode != 0: + results.append( + { + **row, + "status": "failed", + "returncode": commit_result.returncode, + "detail": "git commit failed", + "stdout": compact_output(commit_result.stdout), + "stderr": compact_output(commit_result.stderr), + } + ) + continue + after = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + results.append( + { + **row, + "status": "committed", + "returncode": commit_result.returncode, + "detail": f"created commit {after.head or 'HEAD'}", + "after_head": after.head, + "after_ahead": int(after.ahead or 0), + "after_dirty_count": len(after.dirty_entries), + "stdout": compact_output(commit_result.stdout), + "stderr": compact_output(commit_result.stderr), + } + ) + + if any(item["status"] in {"blocked", "failed"} for item in results): + status = "blocked" if not apply else "partial" + elif any(item["status"] == "committed" for item in results): + status = "committed" + elif any(item["status"] == "planned" for item in results): + status = "planned" + else: + status = "noop" + return {"status": status, "apply": apply, "repositories": results} + + +def resolve_message(*, repo: str, version: str | None, message: str | None) -> str: + if message and message.strip(): + return message.strip() + target = f"v{version.removeprefix('v')}" if version else "release" + return f"Release {repo} {target}" + + +def run(command: tuple[str, ...], *, cwd: Path) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run(command, cwd=cwd, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess(command, 124, exc.stdout or "", exc.stderr or "git command timed out") diff --git a/tools/release/govoplan_release/repository_push.py b/tools/release/govoplan_release/repository_push.py new file mode 100644 index 0000000..1faf3d5 --- /dev/null +++ b/tools/release/govoplan_release/repository_push.py @@ -0,0 +1,149 @@ +"""Preview and run plain Git pushes for selected repositories.""" + +from __future__ import annotations + +from pathlib import Path +import shlex +import subprocess + +from .git_state import collect_repository_snapshot +from .workspace import load_repository_specs, resolve_repo_path, resolve_workspace_root + + +def push_repositories( + *, + repos: tuple[str, ...], + workspace_root: Path | str | None = None, + remote: str = "origin", + apply: bool = False, # noqa: A002 - mirrors API field. +) -> dict[str, object]: + workspace = resolve_workspace_root(workspace_root) + remote = remote.strip() or "origin" + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + selected = tuple(dict.fromkeys(repos)) + results: list[dict[str, object]] = [] + + for repo in selected: + spec = specs.get(repo) + if spec is None: + results.append({"repo": repo, "status": "blocked", "detail": "repository is not listed in repositories.json"}) + continue + snapshot = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + has_upstream = bool(snapshot.upstream) + ahead = int(snapshot.ahead or 0) + behind = int(snapshot.behind or 0) + dirty_count = len(snapshot.dirty_entries) + command = push_command(snapshot.branch, remote=remote, has_upstream=has_upstream) + row: dict[str, object] = { + "repo": repo, + "cwd": snapshot.absolute_path, + "branch": snapshot.branch, + "upstream": snapshot.upstream, + "head": snapshot.head, + "ahead": ahead, + "behind": behind, + "dirty_count": dirty_count, + "push_target": push_target(snapshot.branch, upstream=snapshot.upstream, remote=remote), + "command": command_text(command), + } + if not snapshot.exists or not snapshot.is_git: + results.append({**row, "status": "blocked", "detail": "repository is missing or is not a Git repository"}) + continue + if snapshot.safe_directory_required: + results.append({**row, "status": "blocked", "detail": "Git safe.directory approval is required"}) + continue + if not snapshot.has_head: + results.append({**row, "status": "blocked", "detail": "repository has no commit to push"}) + continue + if not snapshot.branch: + results.append({**row, "status": "blocked", "detail": "repository is not on a named branch"}) + continue + if behind: + results.append({**row, "status": "blocked", "detail": f"repository is behind {snapshot.upstream}"}) + continue + if not apply: + if has_upstream: + status = "planned" if ahead else "noop" + detail = ( + f"{ahead} commit(s) will be pushed to {row['push_target']}" + if ahead + else "branch has no unpushed commits; nothing would be pushed" + ) + else: + status = "planned" + detail = f"branch has no upstream; HEAD will be pushed to {row['push_target']}" + if dirty_count: + detail = f"{detail}; {dirty_count} dirty/untracked path(s) stay local" + results.append({**row, "status": status, "detail": detail}) + continue + if has_upstream and ahead == 0: + detail = "branch has no unpushed commits; nothing was pushed" + if dirty_count: + detail = f"{detail}; {dirty_count} dirty/untracked path(s) stay local" + results.append({**row, "status": "noop", "detail": detail, "command_ran": False}) + continue + + process = run(command, cwd=resolve_repo_path(spec, workspace)) + status = "pushed" if process.returncode == 0 else "failed" + detail = ( + f"pushed {ahead} commit(s) to {row['push_target']}" + if process.returncode == 0 and has_upstream + else f"pushed branch to {row['push_target']}" + if process.returncode == 0 + else "git push failed" + ) + if process.returncode == 0 and dirty_count: + detail = f"{detail}; {dirty_count} dirty/untracked path(s) stay local" + results.append( + { + **row, + "status": status, + "returncode": process.returncode, + "detail": detail, + "command_ran": True, + "stdout": compact_output(process.stdout), + "stderr": compact_output(process.stderr), + } + ) + + if any(item["status"] in {"blocked", "failed"} for item in results): + status = "blocked" if not apply else "partial" + elif any(item["status"] == "pushed" for item in results): + status = "pushed" + elif any(item["status"] == "planned" for item in results): + status = "planned" + else: + status = "noop" + return {"status": status, "apply": apply, "remote": remote, "repositories": results} + + +def push_command(branch: str | None, *, remote: str, has_upstream: bool) -> tuple[str, ...]: + if has_upstream: + return ("git", "push") + if branch: + return ("git", "push", "-u", remote, "HEAD") + return ("git", "push", remote) + + +def push_target(branch: str | None, *, upstream: str | None, remote: str) -> str: + if upstream: + return upstream + if branch: + return f"{remote}/{branch}" + return remote + + +def command_text(command: tuple[str, ...]) -> str: + return " ".join(shlex.quote(part) for part in command) + + +def run(command: tuple[str, ...], *, cwd: Path) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run(command, cwd=cwd, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess(command, 124, exc.stdout or "", exc.stderr or "git push timed out") + + +def compact_output(value: str) -> str: + text = value.strip() + return text if len(text) <= 4000 else f"{text[:4000]}\n... truncated ..." diff --git a/tools/release/govoplan_release/repository_sync.py b/tools/release/govoplan_release/repository_sync.py new file mode 100644 index 0000000..125ce22 --- /dev/null +++ b/tools/release/govoplan_release/repository_sync.py @@ -0,0 +1,99 @@ +"""Fetch remote refs for selected repositories without merging.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess + +from .git_state import collect_repository_snapshot +from .repository_push import command_text, compact_output +from .workspace import load_repository_specs, resolve_repo_path, resolve_workspace_root + + +def sync_repositories( + *, + repos: tuple[str, ...], + workspace_root: Path | str | None = None, + remote: str = "origin", + apply: bool = False, # noqa: A002 - mirrors API field. +) -> dict[str, object]: + workspace = resolve_workspace_root(workspace_root) + remote = remote.strip() or "origin" + specs = {spec.name: spec for spec in load_repository_specs(include_website=False)} + selected = tuple(dict.fromkeys(repos)) + results: list[dict[str, object]] = [] + command = ("git", "fetch", "--prune", "--tags", remote) + + for repo in selected: + spec = specs.get(repo) + if spec is None: + results.append({"repo": repo, "status": "blocked", "detail": "repository is not listed in repositories.json"}) + continue + before = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + row: dict[str, object] = { + "repo": repo, + "cwd": before.absolute_path, + "branch": before.branch, + "upstream": before.upstream, + "head": before.head, + "ahead": int(before.ahead or 0), + "behind": int(before.behind or 0), + "dirty_count": len(before.dirty_entries), + "remote": remote, + "command": command_text(command), + } + if not before.exists or not before.is_git: + results.append({**row, "status": "blocked", "detail": "repository is missing or is not a Git repository"}) + continue + if before.safe_directory_required: + results.append({**row, "status": "blocked", "detail": "Git safe.directory approval is required"}) + continue + if not apply: + results.append({**row, "status": "planned", "detail": "fetch remote refs and tags; no merge or rebase will be run"}) + continue + + process = run(command, cwd=resolve_repo_path(spec, workspace)) + if process.returncode != 0: + results.append( + { + **row, + "status": "failed", + "returncode": process.returncode, + "detail": "git fetch failed", + "stdout": compact_output(process.stdout), + "stderr": compact_output(process.stderr), + } + ) + continue + + after = collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + results.append( + { + **row, + "status": "synced", + "returncode": process.returncode, + "detail": "remote refs and tags fetched; local branch was not merged or rebased", + "after_ahead": int(after.ahead or 0), + "after_behind": int(after.behind or 0), + "after_dirty_count": len(after.dirty_entries), + "stdout": compact_output(process.stdout), + "stderr": compact_output(process.stderr), + } + ) + + if any(item["status"] in {"blocked", "failed"} for item in results): + status = "blocked" if not apply else "partial" + elif any(item["status"] == "synced" for item in results): + status = "synced" + elif any(item["status"] == "planned" for item in results): + status = "planned" + else: + status = "noop" + return {"status": status, "apply": apply, "remote": remote, "repositories": results} + + +def run(command: tuple[str, ...], *, cwd: Path) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run(command, cwd=cwd, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess(command, 124, exc.stdout or "", exc.stderr or "git fetch timed out") diff --git a/tools/release/govoplan_release/selective_catalog.py b/tools/release/govoplan_release/selective_catalog.py new file mode 100644 index 0000000..ae1c74e --- /dev/null +++ b/tools/release/govoplan_release/selective_catalog.py @@ -0,0 +1,407 @@ +"""Selective release catalog candidate generation.""" + +from __future__ import annotations + +import base64 +from datetime import UTC, datetime, timedelta +import json +from pathlib import Path +import re +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from govoplan_core.core.module_package_catalog import validate_module_package_catalog + +from .catalog import DEFAULT_PUBLIC_BASE_URL, canonical_hash, fetch_json +from .contracts import collect_contracts +from .git_state import collect_repository_snapshot +from .model import CatalogEntryChange, SelectiveCatalogCandidate +from .module_directory import write_module_directory +from .workspace import load_repository_specs, resolve_workspace_root, website_root + +GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas" + + +def build_selective_catalog_candidate( + *, + repo_versions: dict[str, str], + channel: str = "stable", + workspace_root: Path | str | None = None, + output_dir: Path | str | None = None, + base_catalog: Path | str | None = None, + signing_keys: tuple[str, ...] = (), + public_base_url: str = DEFAULT_PUBLIC_BASE_URL, + repository_base: str = GITEA_BASE, + expires_days: int = 90, + sequence: int | None = None, + check_public: bool = True, + write_summary: bool = True, +) -> SelectiveCatalogCandidate: + if not repo_versions: + raise ValueError("At least one repo version must be provided.") + parsed_keys = tuple(parse_signing_key(value) for value in signing_keys) + if not parsed_keys: + raise ValueError("At least one signing key is required.") + + workspace = resolve_workspace_root(workspace_root) + web_root = website_root(workspace) + source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel) + payload = read_catalog(source_catalog) + if not isinstance(payload, dict): + raise ValueError("Selective catalog updates require object-style catalogs.") + + generated_at = datetime.now(tz=UTC) + resolved_sequence = sequence or next_sequence(payload, generated_at=generated_at) + candidate = json.loads(json.dumps(payload)) + candidate["channel"] = channel + candidate["sequence"] = resolved_sequence + candidate["generated_at"] = json_datetime(generated_at) + candidate["expires_at"] = json_datetime(generated_at + timedelta(days=expires_days)) + release = candidate.get("release") + if not isinstance(release, dict): + release = {} + release["catalog_url"] = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json" + release["keyring_url"] = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json" + release["selected_units"] = [{"repo": repo, "version": version, "tag": f"v{version.removeprefix('v')}"} for repo, version in sorted(repo_versions.items())] + candidate["release"] = release + + repo_contracts = contracts_by_repo(workspace) + changes = apply_repo_updates( + candidate, + repo_versions=repo_versions, + repo_contracts=repo_contracts, + repository_base=repository_base.rstrip("/"), + ) + + candidate.pop("signature", None) + candidate.pop("signatures", None) + candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys] + + output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at) + catalog_path = output_root / "channels" / f"{channel}.json" + keyring_path = output_root / "keyring.json" + summary_path = output_root / "summary.json" if write_summary else None + catalog_path.parent.mkdir(parents=True, exist_ok=True) + catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + keyring_payload = keyring_payload_for_candidate( + existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json", + signing_keys=parsed_keys, + generated_at=generated_at, + ) + keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + module_directory_files = write_module_directory( + catalog_payload=candidate, + keyring_payload=keyring_payload, + output_root=output_root, + channel=channel, + public_base_url=public_base_url, + ) + + trusted_keys = trusted_keys_from_keyring(keyring_payload) + validation = validate_module_package_catalog( + catalog_path, + require_trusted=True, + approved_channels=(channel,), + trusted_keys=trusted_keys, + ) + validation_warnings = tuple(str(item) for item in validation.get("warnings") or ()) + validation_error = validation.get("error") + + candidate_catalog_hash = canonical_hash(candidate) + candidate_keyring_hash = canonical_hash(keyring_payload) + public_catalog_url = f"{public_base_url.rstrip('/')}/catalogs/v1/channels/{channel}.json" + public_keyring_url = f"{public_base_url.rstrip('/')}/catalogs/v1/keyring.json" + published_catalog_hash: str | None = None + published_keyring_hash: str | None = None + if check_public: + published_catalog = fetch_json(public_catalog_url) + if published_catalog["ok"]: + published_catalog_hash = canonical_hash(published_catalog["payload"]) + published_keyring = fetch_json(public_keyring_url) + if published_keyring["ok"]: + published_keyring_hash = canonical_hash(published_keyring["payload"]) + + result = SelectiveCatalogCandidate( + generated_at=json_datetime(generated_at), + channel=channel, + status="ready" if validation.get("valid") else "blocked", + candidate_dir=str(output_root), + catalog_path=str(catalog_path), + keyring_path=str(keyring_path), + summary_path=str(summary_path) if summary_path is not None else None, + source_catalog=str(source_catalog), + public_catalog_url=public_catalog_url, + public_keyring_url=public_keyring_url, + sequence=resolved_sequence, + signature_count=len(candidate["signatures"]) if isinstance(candidate.get("signatures"), list) else 0, + key_count=len(keyring_payload.get("keys", ())) if isinstance(keyring_payload.get("keys"), list) else 0, + candidate_catalog_hash=candidate_catalog_hash, + candidate_keyring_hash=candidate_keyring_hash, + published_catalog_hash=published_catalog_hash, + published_keyring_hash=published_keyring_hash, + candidate_matches_published_catalog=compare_hash(candidate_catalog_hash, published_catalog_hash), + candidate_matches_published_keyring=compare_hash(candidate_keyring_hash, published_keyring_hash), + validation_valid=validation.get("valid") is True, + validation_error=str(validation_error) if validation_error else None, + validation_warnings=validation_warnings, + changes=tuple(changes), + notes=( + "Candidate catalog preserves unchanged entries from the source catalog.", + f"Generated {len(module_directory_files)} module-directory file(s).", + "Publishing is intentionally separate from candidate generation.", + ), + ) + if summary_path is not None: + summary_path.write_text(json.dumps(dataclass_payload(result), indent=2, sort_keys=True) + "\n", encoding="utf-8") + return result + + +def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str: + if base_catalog is not None: + value = str(base_catalog) + return value if value.startswith(("http://", "https://")) else Path(value).expanduser() + local = web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json" + if local.exists(): + return local + return f"{DEFAULT_PUBLIC_BASE_URL}/catalogs/v1/channels/{channel}.json" + + +def read_catalog(source: Path | str) -> object: + if isinstance(source, str) and source.startswith(("http://", "https://")): + payload = fetch_json(source) + if not payload["ok"]: + raise ValueError(f"Could not fetch catalog {source}: {payload['error']}") + return payload["payload"] + return json.loads(Path(source).read_text(encoding="utf-8")) + + +def next_sequence(payload: dict[str, Any], *, generated_at: datetime) -> int: + timestamp_sequence = int(generated_at.strftime("%Y%m%d%H%M")) + try: + current = int(payload.get("sequence") or 0) + except (TypeError, ValueError): + current = 0 + return max(current + 1, timestamp_sequence) + + +def contracts_by_repo(workspace: Path) -> dict[str, Any]: + specs = load_repository_specs(include_website=False) + snapshots = tuple( + collect_repository_snapshot(spec, workspace_root=workspace, target_tag=None, online=False) + for spec in specs + ) + return {contract.repo: contract for contract in collect_contracts(snapshots)} + + +def apply_repo_updates( + payload: dict[str, Any], + *, + repo_versions: dict[str, str], + repo_contracts: dict[str, Any], + repository_base: str, +) -> list[CatalogEntryChange]: + changes: list[CatalogEntryChange] = [] + modules = payload.get("modules") + if not isinstance(modules, list): + raise ValueError("Catalog payload has no modules list.") + + handled: set[str] = set() + if "govoplan-core" in repo_versions: + version = repo_versions["govoplan-core"].removeprefix("v") + tag = f"v{version}" + core_release = payload.get("core_release") + if not isinstance(core_release, dict): + raise ValueError("Catalog payload has no core_release object.") + changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="version", value=version)) + changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="python_ref", value=f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}")) + changes.extend(update_field(core_release, repo="govoplan-core", module_id=None, field="webui_ref", value=f"{repository_base}/govoplan-core.git#{tag}")) + release = payload.get("release") + if isinstance(release, dict) and isinstance(release.get("version"), str): + changes.extend(update_field(release, repo="govoplan-core", module_id=None, field="version", value=version)) + changes.extend(update_field(release, repo="govoplan-core", module_id=None, field="tag", value=tag)) + handled.add("govoplan-core") + + for entry in modules: + if not isinstance(entry, dict): + continue + repo = module_entry_repo(entry) + if repo not in repo_versions: + continue + version = repo_versions[repo].removeprefix("v") + tag = f"v{version}" + module_id = str(entry.get("module_id") or "") + package = str(entry.get("python_package") or repo) + changes.extend(update_field(entry, repo=repo, module_id=module_id, field="version", value=version)) + changes.extend(update_field(entry, repo=repo, module_id=module_id, field="python_ref", value=f"{package} @ {repository_base}/{repo}.git@{tag}")) + if entry.get("webui_package"): + changes.extend(update_field(entry, repo=repo, module_id=module_id, field="webui_ref", value=f"{repository_base}/{repo}.git#{tag}")) + contract = repo_contracts.get(repo) + if contract is not None: + provider_payload = [{"name": item.name, "version": item.version} for item in contract.provides_interfaces] + requirement_payload = [ + { + "name": item.name, + **({"version_min": item.version_min} if item.version_min else {}), + **({"version_max_exclusive": item.version_max_exclusive} if item.version_max_exclusive else {}), + "optional": item.optional, + } + for item in contract.requires_interfaces + ] + if provider_payload: + changes.extend(update_json_field(entry, repo=repo, module_id=module_id, field="provides_interfaces", value=provider_payload)) + if requirement_payload: + changes.extend(update_json_field(entry, repo=repo, module_id=module_id, field="requires_interfaces", value=requirement_payload)) + handled.add(repo) + + missing = sorted(set(repo_versions) - handled) + if missing: + raise ValueError(f"Selected repositories are not represented in the catalog: {', '.join(missing)}") + return changes + + +def module_entry_repo(entry: dict[str, Any]) -> str | None: + for field in ("python_ref", "webui_ref"): + value = entry.get(field) + if isinstance(value, str): + match = re.search(r"/([^/@#]+)[.]git(?:[@#]|$)", value) + if match: + return match.group(1) + package = entry.get("python_package") + return str(package).split("[", 1)[0] if isinstance(package, str) and package.startswith("govoplan-") else None + + +def update_field(target: dict[str, Any], *, repo: str, module_id: str | None, field: str, value: str) -> list[CatalogEntryChange]: + before = target.get(field) + before_text = before if isinstance(before, str) else None + if before_text == value: + return [] + target[field] = value + return [CatalogEntryChange(repo=repo, module_id=module_id, field=field, before=before_text, after=value)] + + +def update_json_field(target: dict[str, Any], *, repo: str, module_id: str | None, field: str, value: list[dict[str, object]]) -> list[CatalogEntryChange]: + before = target.get(field) + if before == value: + return [] + target[field] = value + return [ + CatalogEntryChange( + repo=repo, + module_id=module_id, + field=field, + before=json.dumps(before, sort_keys=True) if before is not None else None, + after=json.dumps(value, sort_keys=True), + ) + ] + + +def parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]: + key_id, separator, path_text = value.partition("=") + if not separator or not key_id.strip() or not path_text.strip(): + raise ValueError("--catalog-signing-key must use KEY_ID=/path/to/private.pem") + path = Path(path_text).expanduser() + private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(private_key, Ed25519PrivateKey): + raise ValueError(f"Catalog signing key must be an Ed25519 private key: {path}") + return key_id.strip(), private_key + + +def signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]: + signature_payload = dict(payload) + signature_payload.pop("signature", None) + signature_payload.pop("signatures", None) + return { + "algorithm": "ed25519", + "key_id": key_id, + "value": base64.b64encode(private_key.sign(canonical_bytes(signature_payload))).decode("ascii"), + } + + +def keyring_payload_for_candidate( + *, + existing_keyring: Path, + signing_keys: tuple[tuple[str, Ed25519PrivateKey], ...], + generated_at: datetime, +) -> dict[str, Any]: + expected = {key_id: public_key_base64(private_key) for key_id, private_key in signing_keys} + if existing_keyring.exists(): + payload = json.loads(existing_keyring.read_text(encoding="utf-8")) + keys = payload.get("keys") if isinstance(payload, dict) else None + if isinstance(keys, list): + existing = { + str(item.get("key_id")): str(item.get("public_key") or item.get("public_key_base64")) + for item in keys + if isinstance(item, dict) and item.get("key_id") + } + if all(existing.get(key_id) == public_key for key_id, public_key in expected.items()): + return payload + return { + "keyring_version": "1", + "purpose": "govoplan module package catalog signatures", + "generated_at": json_datetime(generated_at), + "keys": [ + { + "key_id": key_id, + "status": "active", + "public_key": public_key_base64(private_key), + "not_before": generated_at.date().isoformat() + "T00:00:00Z", + } + for key_id, private_key in signing_keys + ], + } + + +def trusted_keys_from_keyring(payload: dict[str, Any]) -> dict[str, str]: + keys = payload.get("keys") + if not isinstance(keys, list): + return {} + result: dict[str, str] = {} + for item in keys: + if not isinstance(item, dict): + continue + status = str(item.get("status") or "active").lower() + if status in {"revoked", "disabled", "retired"}: + continue + key_id = str(item.get("key_id") or "") + public_key = str(item.get("public_key") or item.get("public_key_base64") or "") + if key_id and public_key: + result[key_id] = public_key + return result + + +def public_key_base64(private_key: Ed25519PrivateKey) -> str: + public_bytes = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + return base64.b64encode(public_bytes).decode("ascii") + + +def canonical_bytes(payload: object) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def compare_hash(candidate_hash: str | None, published_hash: str | None) -> bool | None: + if candidate_hash is None or published_hash is None: + return None + return candidate_hash == published_hash + + +def resolve_output_dir(*, output_dir: Path | str | None, channel: str, generated_at: datetime) -> Path: + if output_dir is not None: + return Path(output_dir).expanduser() + stamp = generated_at.strftime("%Y%m%d-%H%M%S") + return Path.cwd() / "runtime" / "release-candidates" / f"{channel}-{stamp}" + + +def json_datetime(value: datetime) -> str: + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def dataclass_payload(value: object) -> dict[str, object]: + from dataclasses import asdict + + return asdict(value) diff --git a/tools/release/govoplan_release/selective_planner.py b/tools/release/govoplan_release/selective_planner.py new file mode 100644 index 0000000..b870eb1 --- /dev/null +++ b/tools/release/govoplan_release/selective_planner.py @@ -0,0 +1,271 @@ +"""Selective release planning for independently versioned packages.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import shlex + +from .contracts import validate_contracts +from .model import ( + CompatibilityIssue, + InterfaceProviderSnapshot, + InterfaceRequirementSnapshot, + ReleaseDashboard, + ReleasePlanStep, + ReleasePlanUnit, + RepositorySnapshot, + SelectiveReleasePlan, + ModuleContractSnapshot, +) + + +def build_selective_release_plan( + dashboard: ReleaseDashboard, + *, + selected_repos: tuple[str, ...] = (), + target_version: str | None = None, + repo_versions: dict[str, str] | None = None, + channel: str = "stable", +) -> SelectiveReleasePlan: + repo_versions = repo_versions or {} + repositories = selected_repositories(dashboard, selected_repos=selected_repos) + contracts_by_repo = {contract.repo: contract for contract in dashboard.contracts} + units = tuple( + build_unit(repo, target_version=repo_versions.get(repo.spec.name) or target_version, contracts=contracts_by_repo.get(repo.spec.name)) + for repo in repositories + ) + compatibility = compatibility_issues(dashboard) + steps = dry_run_steps(units=units, dashboard=dashboard, channel=channel) + notes = release_notes(dashboard) + status = plan_status(units=units, compatibility=compatibility) + return SelectiveReleasePlan( + generated_at=datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + target_channel=channel, + status=status, + units=units, + compatibility=compatibility, + dry_run_steps=steps, + notes=notes, + ) + + +def selected_repositories(dashboard: ReleaseDashboard, *, selected_repos: tuple[str, ...]) -> tuple[RepositorySnapshot, ...]: + if selected_repos: + wanted = set(selected_repos) + return tuple(repo for repo in dashboard.repositories if repo.spec.name in wanted) + return tuple(repo for repo in dashboard.repositories if release_candidate(repo)) + + +def release_candidate(repo: RepositorySnapshot) -> bool: + return repo.dirty or bool(repo.ahead) or (repo.exists and repo.is_git and not repo.has_head) + + +def build_unit(repo: RepositorySnapshot, *, target_version: str | None, contracts: ModuleContractSnapshot | None) -> ReleasePlanUnit: + current_version = repo.versions.primary + resolved_target = (target_version or current_version or "0.1.0").removeprefix("v") + target_tag = f"v{resolved_target}" + blockers: list[str] = [] + warnings: list[str] = [] + if not repo.exists: + blockers.append("repository is missing locally") + elif not repo.is_git: + blockers.append("path is not a Git repository") + elif repo.safe_directory_required: + blockers.append("Git safe.directory approval is required") + elif repo.behind: + blockers.append(f"repository is behind {repo.upstream}") + if repo.exists and repo.is_git and not repo.has_head: + blockers.append("repository has no initial commit") + if repo.dirty: + warnings.append("uncommitted changes will need review before release") + if repo.ahead: + warnings.append("unpushed commits should be pushed before catalog publication") + if current_version is None: + warnings.append("no local version metadata found; initial release needs version metadata and catalog entry synthesis") + if current_version and current_version == resolved_target and repo.local_target_tag_exists: + warnings.append(f"local tag {target_tag} already exists") + + provides: tuple[InterfaceProviderSnapshot, ...] = () + requires: tuple[InterfaceRequirementSnapshot, ...] = () + if contracts is not None: + provides = contracts.provides_interfaces + requires = contracts.requires_interfaces + + return ReleasePlanUnit( + repo=repo.spec.name, + category=repo.spec.category, + subtype=repo.spec.subtype, + branch=repo.branch, + current_version=current_version, + target_version=resolved_target, + target_tag=target_tag, + status="blocked" if blockers else "attention" if warnings else "ready", + blockers=tuple(blockers), + warnings=tuple(warnings), + provides_interfaces=provides, + requires_interfaces=requires, + ) + + +def compatibility_issues(dashboard: ReleaseDashboard) -> tuple[CompatibilityIssue, ...]: + return validate_contracts(dashboard.contracts) + + +def dry_run_steps(*, units: tuple[ReleasePlanUnit, ...], dashboard: ReleaseDashboard, channel: str) -> tuple[ReleasePlanStep, ...]: + steps: list[ReleasePlanStep] = [] + for unit in units: + steps.append( + ReleasePlanStep( + id=f"{unit.repo}:preflight", + title=f"Inspect {unit.repo}", + detail="Verify local branch, dirty state, current version, and target tag before release.", + command="git status --short --branch && git tag --list " + shlex.quote(unit.target_tag), + cwd=f"{dashboard.workspace_root}/{unit.repo}", + repo=unit.repo, + ) + ) + if unit.current_version != unit.target_version: + steps.append( + ReleasePlanStep( + id=f"{unit.repo}:version", + title=f"Update {unit.repo} version metadata", + detail=f"Prepare version files and manifest metadata for {unit.target_version}.", + command=None, + cwd=f"{dashboard.workspace_root}/{unit.repo}", + mutating=True, + repo=unit.repo, + status="needs-executor", + ) + ) + steps.append( + ReleasePlanStep( + id=f"{unit.repo}:commit", + title=f"Commit {unit.repo} release changes", + detail="Commit reviewed changes for this release unit only.", + command=f"git add -A && git commit -m {shlex.quote(f'Release {unit.repo} {unit.target_tag}')}", + cwd=f"{dashboard.workspace_root}/{unit.repo}", + mutating=True, + repo=unit.repo, + ) + ) + steps.append( + ReleasePlanStep( + id=f"{unit.repo}:tag", + title=f"Create {unit.repo} tag {unit.target_tag}", + detail="Create an annotated tag for this package release.", + command=f"git tag -a {shlex.quote(unit.target_tag)} -m {shlex.quote(f'Release {unit.repo} {unit.target_tag}')}", + cwd=f"{dashboard.workspace_root}/{unit.repo}", + mutating=True, + repo=unit.repo, + ) + ) + steps.append( + ReleasePlanStep( + id=f"{unit.repo}:push", + title=f"Push {unit.repo} release", + detail="Push the branch and release tag for this package only.", + command=f"git push --atomic origin HEAD:refs/heads/{shlex.quote(unit.branch or 'main')} refs/tags/{shlex.quote(unit.target_tag)}", + cwd=f"{dashboard.workspace_root}/{unit.repo}", + mutating=True, + repo=unit.repo, + ) + ) + + if units: + initial_units = tuple(unit for unit in units if unit.current_version is None) + repo_version_args = " ".join( + f"--repo-version {shlex.quote(unit.repo + '=' + unit.target_version)}" + for unit in units + ) + if initial_units: + steps.append( + ReleasePlanStep( + id="catalog:initial-entry-synthesis", + title=f"Prepare initial {channel} catalog entries", + detail=( + "Selected initial-release repositories are not guaranteed to exist in the channel catalog yet: " + + ", ".join(unit.repo for unit in initial_units) + + ". The current selective catalog writer updates existing entries only." + ), + cwd=dashboard.meta_root, + mutating=True, + status="needs-catalog-entry-synthesis", + ) + ) + else: + steps.append( + ReleasePlanStep( + id="catalog:selective-generator", + title=f"Update {channel} release channel catalog", + detail=( + "Generate a signed candidate catalog that preserves unchanged package versions " + "and updates only selected release units." + ), + command=( + "KEY_DIR=\"$HOME/.config/govoplan/release-keys\" " + f"tools/release/release-catalog.py selective --channel {shlex.quote(channel)} " + f"{repo_version_args} " + "--catalog-signing-key \"release-key-1=$KEY_DIR/release-key-1.pem\"" + ), + cwd=dashboard.meta_root, + mutating=True, + status="planned", + ) + ) + steps.append( + ReleasePlanStep( + id="catalog:validate-sign-publish", + title=f"Validate, sign, and publish {channel}", + detail="The candidate writer validates and signs locally. Publishing to the website repo remains a separate apply step.", + command="Review the candidate under runtime/release-candidates before publishing it to the website repository.", + cwd=dashboard.meta_root, + status="needs-publish-executor", + ) + ) + return tuple(steps) + + +def release_notes(dashboard: ReleaseDashboard) -> tuple[str, ...]: + notes = [ + "Release units are independent; selecting one package should not force every repository to a shared tag.", + "Channel publication must preserve unchanged package versions and only advance selected release units.", + "Compatibility is checked from currently discovered manifest interface providers and requirements.", + ] + if dashboard.catalog.public_checked: + notes.append("Local catalog/keyring hashes are compared with the published channel and keyring.") + return tuple(notes) + + +def plan_status(*, units: tuple[ReleasePlanUnit, ...], compatibility: tuple[CompatibilityIssue, ...]) -> str: + if any(unit.blockers for unit in units) or any(issue.severity == "blocker" for issue in compatibility): + return "blocked" + if units or compatibility: + return "attention" + return "ready" + + +def version_satisfies(version: str, requirement: InterfaceRequirementSnapshot) -> bool: + parsed = parse_version(version) + if parsed is None: + return False + if requirement.version_min and (minimum := parse_version(requirement.version_min)) is not None and parsed < minimum: + return False + if requirement.version_max_exclusive and (maximum := parse_version(requirement.version_max_exclusive)) is not None and parsed >= maximum: + return False + return True + + +def parse_version(value: str) -> tuple[int, ...] | None: + parts = value.split(".") + if not parts or any(not part.isdigit() for part in parts): + return None + return tuple(int(part) for part in parts) + + +def format_requirement(requirement: InterfaceRequirementSnapshot) -> str: + parts: list[str] = [] + if requirement.version_min: + parts.append(f">={requirement.version_min}") + if requirement.version_max_exclusive: + parts.append(f"<{requirement.version_max_exclusive}") + return ", ".join(parts) if parts else "any version" diff --git a/tools/release/govoplan_release/workspace.py b/tools/release/govoplan_release/workspace.py new file mode 100644 index 0000000..68012e5 --- /dev/null +++ b/tools/release/govoplan_release/workspace.py @@ -0,0 +1,59 @@ +"""Workspace and repository configuration helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .model import RepositorySpec + + +META_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_WORKSPACE_ROOT = META_ROOT.parent +REPOSITORIES_FILE = META_ROOT / "repositories.json" + + +def load_repository_specs(*, include_website: bool = False) -> tuple[RepositorySpec, ...]: + payload = json.loads(REPOSITORIES_FILE.read_text(encoding="utf-8")) + repositories = payload.get("repositories") + if not isinstance(repositories, list): + raise ValueError(f"{REPOSITORIES_FILE} does not contain a repositories list") + + specs: list[RepositorySpec] = [] + for item in repositories: + if not isinstance(item, dict): + continue + category = str(item.get("category", "")) + if category == "website" and not include_website: + continue + specs.append( + RepositorySpec( + name=str(item["name"]), + category=category, + subtype=str(item.get("subtype", "")), + remote=str(item.get("remote", "")), + path=str(item["path"]), + ) + ) + return tuple(specs) + + +def resolve_workspace_root(value: Path | str | None = None) -> Path: + if value is None: + return DEFAULT_WORKSPACE_ROOT + return Path(value).expanduser().resolve() + + +def resolve_repo_path(spec: RepositorySpec, workspace_root: Path) -> Path: + path = Path(spec.path) + if path.is_absolute(): + return path + return workspace_root / path + + +def core_root(workspace_root: Path) -> Path: + return workspace_root / "govoplan-core" + + +def website_root(workspace_root: Path) -> Path: + return workspace_root / "addideas-govoplan-website" diff --git a/tools/release/release-catalog.py b/tools/release/release-catalog.py new file mode 100644 index 0000000..24ec6ab --- /dev/null +++ b/tools/release/release-catalog.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Build GovOPlaN release catalog candidates.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from govoplan_release.module_directory import write_module_directory +from govoplan_release.model import to_jsonable +from govoplan_release.publisher import publish_catalog_candidate +from govoplan_release.selective_catalog import build_selective_catalog_candidate +from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + selective = subparsers.add_parser("selective", help="Build a signed selective channel catalog candidate.") + selective.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT) + selective.add_argument("--channel", default="stable") + selective.add_argument("--repo-version", action="append", default=[], metavar="REPO=VERSION", required=True) + selective.add_argument("--catalog-signing-key", action="append", default=[], metavar="KEY_ID=PRIVATE_KEY", required=True) + selective.add_argument("--base-catalog", help="Catalog path or URL to use as the source. Defaults to local channel, then public channel.") + selective.add_argument("--output-dir", type=Path, help="Candidate output directory. Defaults to runtime/release-candidates/-.") + selective.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") + selective.add_argument("--repository-base", default="git+ssh://git@git.add-ideas.de/add-ideas") + selective.add_argument("--expires-days", type=int, default=90) + selective.add_argument("--sequence", type=int) + selective.add_argument("--skip-public-check", action="store_true") + selective.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.") + + publish = subparsers.add_parser("publish-candidate", help="Publish a reviewed catalog candidate into the website repo.") + publish.add_argument("--candidate-dir", type=Path, required=True) + publish.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT) + publish.add_argument("--web-root", type=Path) + publish.add_argument("--channel", default="stable") + publish.add_argument("--apply", action="store_true", help="Copy candidate files into the website repo. Without this, only preview.") + publish.add_argument("--build-web", action="store_true") + publish.add_argument("--commit", action="store_true") + publish.add_argument("--tag", action="store_true", help="Create a website catalog publication tag. Implies --commit.") + publish.add_argument("--push", action="store_true", help="Push the website branch and tag. Implies --commit.") + publish.add_argument("--remote", default="origin") + publish.add_argument("--branch") + publish.add_argument("--tag-name") + publish.add_argument("--npm", default="npm") + publish.add_argument("--allow-dirty-website", action="store_true") + publish.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.") + + directory = subparsers.add_parser("module-directory", help="Generate browsable module-directory files from a catalog and keyring.") + directory.add_argument("--catalog", type=Path, required=True) + directory.add_argument("--keyring", type=Path, required=True) + directory.add_argument("--output-dir", type=Path, required=True) + directory.add_argument("--channel", default="stable") + directory.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") + directory.add_argument("--json", action="store_true", help="Print machine-readable summary JSON.") + + args = parser.parse_args() + if args.command == "selective": + result = build_selective_catalog_candidate( + repo_versions=parse_repo_versions(tuple(args.repo_version)), + channel=args.channel, + workspace_root=args.workspace_root, + output_dir=args.output_dir, + base_catalog=args.base_catalog, + signing_keys=tuple(args.catalog_signing_key), + public_base_url=args.public_base_url, + repository_base=args.repository_base, + expires_days=args.expires_days, + sequence=args.sequence, + check_public=not args.skip_public_check, + ) + payload = to_jsonable(result) + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print_text_summary(payload) + return 0 if result.status == "ready" else 1 + if args.command == "publish-candidate": + result = publish_catalog_candidate( + candidate_dir=args.candidate_dir, + workspace_root=args.workspace_root, + web_root=args.web_root, + channel=args.channel, + apply=args.apply, + build_web=args.build_web, + commit=args.commit, + tag=args.tag, + push=args.push, + remote=args.remote, + branch=args.branch, + npm=args.npm, + tag_name=args.tag_name, + allow_dirty_website=args.allow_dirty_website, + ) + payload = to_jsonable(result) + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print_publish_summary(payload) + return 0 if result.status in {"ready", "applied", "published"} else 1 + if args.command == "module-directory": + catalog_payload = json.loads(args.catalog.read_text(encoding="utf-8")) + keyring_payload = json.loads(args.keyring.read_text(encoding="utf-8")) + files = write_module_directory( + catalog_payload=catalog_payload, + keyring_payload=keyring_payload, + output_root=args.output_dir, + channel=args.channel, + public_base_url=args.public_base_url, + ) + payload = {"status": "generated", "output_dir": str(args.output_dir), "files": [str(path) for path in files]} + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print("Module directory") + print(f" output_dir: {args.output_dir}") + print(f" files: {len(files)}") + for path in files[:20]: + print(f" {path}") + if len(files) > 20: + print(f" ... {len(files) - 20} more") + return 0 + return 2 + + +def parse_repo_versions(values: tuple[str, ...]) -> dict[str, str]: + result: dict[str, str] = {} + for value in values: + if "=" not in value: + raise SystemExit(f"--repo-version must use REPO=VERSION: {value}") + repo, version = value.split("=", 1) + repo = repo.strip() + version = version.strip().removeprefix("v") + if not repo or not version: + raise SystemExit(f"--repo-version must use REPO=VERSION: {value}") + result[repo] = version + return result + + +def print_text_summary(payload: dict[str, object]) -> None: + print("Selective catalog candidate") + for key in ( + "status", + "channel", + "sequence", + "catalog_path", + "keyring_path", + "summary_path", + "validation_valid", + "validation_error", + "candidate_matches_published_catalog", + "candidate_matches_published_keyring", + ): + print(f" {key}: {payload.get(key)}") + changes = payload.get("changes") + if isinstance(changes, list): + print(f" changes: {len(changes)}") + for item in changes[:20]: + if not isinstance(item, dict): + continue + print( + " " + f"{item.get('repo')}:{item.get('module_id') or '-'} " + f"{item.get('field')} {item.get('before')!r} -> {item.get('after')!r}" + ) + if len(changes) > 20: + print(f" ... {len(changes) - 20} more") + warnings = payload.get("validation_warnings") + if isinstance(warnings, list) and warnings: + print(" validation warnings:") + for warning in warnings: + print(f" {warning}") + + +def print_publish_summary(payload: dict[str, object]) -> None: + print("Catalog publish candidate") + for key in ( + "status", + "applied", + "channel", + "candidate_dir", + "web_root", + "target_catalog_path", + "target_keyring_path", + "branch", + "tag_name", + "validation_valid", + "validation_error", + "catalog_changed", + "keyring_changed", + ): + print(f" {key}: {payload.get(key)}") + steps = payload.get("steps") + if isinstance(steps, list): + print(" steps:") + for item in steps: + if not isinstance(item, dict): + continue + print(f" [{item.get('status')}] {item.get('title')}") + command = item.get("command") + if command: + print(f" {command}") + notes = payload.get("notes") + if isinstance(notes, list) and notes: + print(" notes:") + for note in notes: + print(f" {note}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/release-console.py b/tools/release/release-console.py new file mode 100644 index 0000000..5d6eeb9 --- /dev/null +++ b/tools/release/release-console.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Start the local GovOPlaN release console.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import secrets +import sys + +from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="127.0.0.1", help="Bind host. Defaults to 127.0.0.1.") + parser.add_argument("--port", type=int, default=8765, help="Bind port. Defaults to 8765.") + parser.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT) + parser.add_argument("--no-token", action="store_true", help="Disable the local API token guard.") + args = parser.parse_args() + + try: + import uvicorn + except ImportError: + print("uvicorn is required. Install the meta development environment with `pip install -r requirements-dev.txt`.", file=sys.stderr) + return 2 + + from server.app import create_app + + workspace_root = args.workspace_root.expanduser().resolve() + token = None if args.no_token else secrets.token_urlsafe(24) + app = create_app(workspace_root=workspace_root, token=token) + url = f"http://{args.host}:{args.port}/" + if token: + url = f"{url}?token={token}" + print("GovOPlaN release console") + print(f" workspace: {workspace_root}") + print(f" url: {url}") + print(" mode: local release dashboard; mutating actions require explicit confirmation") + uvicorn.run(app, host=args.host, port=args.port, log_level="info") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/release-doctor.py b/tools/release/release-doctor.py index 8f72159..be29476 100644 --- a/tools/release/release-doctor.py +++ b/tools/release/release-doctor.py @@ -15,10 +15,8 @@ import subprocess import sys import tomllib from typing import Any -from urllib.error import URLError -from urllib.parse import urlparse -from urllib.request import Request, urlopen +from govoplan_release.http_fetch import fetch_http, validate_http_url META_ROOT = Path(__file__).resolve().parents[2] ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve() @@ -1208,14 +1206,11 @@ def safe_directory_argv(path: Path) -> list[str]: def check_url(url: str) -> dict[str, Any]: - parsed = urlparse(url) - if parsed.scheme not in {"http", "https"} or not parsed.netloc: - return {"ok": False, "error": "URL must use http:// or https:// with a hostname", "url": url} - request = Request(url, method="HEAD") try: - with urlopen(request, timeout=8) as response: # noqa: S310 - validated operator-requested release availability check. # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - return {"ok": 200 <= response.status < 400, "status": response.status, "url": url} - except URLError as exc: + checked_url = validate_http_url(url) + response = fetch_http(checked_url, timeout=8, method="HEAD") + return {"ok": 200 <= response.status < 400, "status": response.status, "url": checked_url} + except (ValueError, OSError) as exc: return {"ok": False, "error": str(exc), "url": url} diff --git a/tools/release/release-plan.py b/tools/release/release-plan.py new file mode 100644 index 0000000..33f18cd --- /dev/null +++ b/tools/release/release-plan.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Generate a dry-run selective GovOPlaN release plan.""" + +from __future__ import annotations + +import argparse +from datetime import UTC, datetime +import json +from pathlib import Path + +from govoplan_release import build_dashboard, build_selective_release_plan +from govoplan_release.model import to_jsonable +from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT, META_ROOT + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace-root", type=Path, default=DEFAULT_WORKSPACE_ROOT) + parser.add_argument("--repo", action="append", default=[], help="Repository to include. May be repeated.") + parser.add_argument("--target-version", help="Default target version for selected repositories.") + parser.add_argument( + "--repo-version", + action="append", + default=[], + metavar="REPO=VERSION", + help="Per-repository target version. May be repeated.", + ) + parser.add_argument("--channel", default="stable", help="Release channel to plan for. Defaults to stable.") + parser.add_argument("--online", action="store_true", help="Check published catalog/keyring.") + parser.add_argument("--remote-tags", action="store_true", help="Also check remote Git tags. This can be slow across many repositories.") + parser.add_argument("--include-migrations", action="store_true", help="Run migration audits before planning.") + parser.add_argument("--output", type=Path, help="Write JSON plan to this path.") + args = parser.parse_args() + + dashboard = build_dashboard( + workspace_root=args.workspace_root, + target_version=args.target_version, + online=args.online, + check_remote_tags=args.remote_tags, + check_public_catalog=args.online, + include_migrations=args.include_migrations, + channel=args.channel, + ) + plan = build_selective_release_plan( + dashboard, + selected_repos=tuple(args.repo), + target_version=args.target_version, + repo_versions=parse_repo_versions(tuple(args.repo_version)), + channel=args.channel, + ) + payload = json.dumps(to_jsonable(plan), indent=2, sort_keys=True) + "\n" + if args.output: + output = args.output.expanduser() + if not output.is_absolute(): + output = META_ROOT / output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(payload, encoding="utf-8") + print(output) + else: + print(payload, end="") + return 1 if plan.status == "blocked" else 0 + + +def parse_repo_versions(values: tuple[str, ...]) -> dict[str, str]: + result: dict[str, str] = {} + for value in values: + if "=" not in value: + raise SystemExit(f"--repo-version must use REPO=VERSION: {value}") + repo, version = value.split("=", 1) + repo = repo.strip() + version = version.strip() + if not repo or not version: + raise SystemExit(f"--repo-version must use REPO=VERSION: {value}") + result[repo] = version + return result + + +def default_plan_path(channel: str) -> Path: + stamp = datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S") + return META_ROOT / "runtime" / "release-plans" / f"{channel}-{stamp}.json" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release/server/__init__.py b/tools/release/server/__init__.py new file mode 100644 index 0000000..e0ca861 --- /dev/null +++ b/tools/release/server/__init__.py @@ -0,0 +1 @@ +"""Local release console server package.""" diff --git a/tools/release/server/app.py b/tools/release/server/app.py new file mode 100644 index 0000000..bd34fd1 --- /dev/null +++ b/tools/release/server/app.py @@ -0,0 +1,331 @@ +"""FastAPI app for the local GovOPlaN release console.""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse +from pydantic import BaseModel, Field + +from govoplan_release import ( + build_dashboard, + build_release_intelligence, + build_release_plan, + build_selective_catalog_candidate, + build_selective_release_plan, + publish_catalog_candidate, + prepare_repositories, + push_repositories, + sync_repositories, +) +from govoplan_release.model import to_jsonable +from govoplan_release.workspace import DEFAULT_WORKSPACE_ROOT, META_ROOT + + +class CatalogCandidateRequest(BaseModel): + repo_versions: dict[str, str] = Field(default_factory=dict) + repos: list[str] = Field(default_factory=list) + target_version: str | None = None + channel: str = "stable" + signing_keys: list[str] = Field(default_factory=list) + output_dir: str | None = None + base_catalog: str | None = None + expires_days: int = 90 + sequence: int | None = None + public_base_url: str = "https://govoplan.add-ideas.de" + repository_base: str = "git+ssh://git@git.add-ideas.de/add-ideas" + check_public: bool = True + + +class PublishCandidateRequest(BaseModel): + candidate_dir: str + channel: str = "stable" + web_root: str | None = None + apply: bool = False # noqa: A003 - API field mirrors CLI. + build_web: bool = False + commit: bool = False + tag: bool = False + push: bool = False + remote: str = "origin" + branch: str | None = None + tag_name: str | None = None + npm: str = "npm" + allow_dirty_website: bool = False + confirm: str = "" + + +class RepositoryPushRequest(BaseModel): + repos: list[str] = Field(default_factory=list) + remote: str = "origin" + apply: bool = False # noqa: A003 - API field mirrors CLI. + confirm: str = "" + + +class RepositorySyncRequest(BaseModel): + repos: list[str] = Field(default_factory=list) + remote: str = "origin" + apply: bool = False # noqa: A003 - API field mirrors CLI. + confirm: str = "" + + +class RepositoryPrepareRequest(BaseModel): + repos: list[str] = Field(default_factory=list) + repo_versions: dict[str, str] = Field(default_factory=dict) + message: str | None = None + apply: bool = False # noqa: A003 - API field mirrors CLI. + confirm: str = "" + + +def create_app(*, workspace_root: Path = DEFAULT_WORKSPACE_ROOT, token: str | None = None) -> FastAPI: + app = FastAPI(title="GovOPlaN Release Console", version="0.1.0") + app.state.workspace_root = workspace_root + app.state.token = token + + @app.middleware("http") + async def require_token(request: Request, call_next): # type: ignore[no-untyped-def] + if token and request.url.path.startswith("/api/"): + provided = request.headers.get("x-release-console-token") or request.query_params.get("token") + if provided != token: + return JSONResponse({"detail": "release console token required"}, status_code=401) + return await call_next(request) + + @app.get("/", response_class=HTMLResponse) + def index() -> HTMLResponse: + index_path = Path(__file__).resolve().parents[1] / "webui" / "index.html" + if not index_path.exists(): + raise HTTPException(status_code=500, detail="release console UI is missing") + return HTMLResponse(index_path.read_text(encoding="utf-8"), headers={"Cache-Control": "no-store"}) + + @app.get("/api/health") + def health() -> dict[str, object]: + return { + "ok": True, + "workspace_root": str(app.state.workspace_root), + "token_required": bool(app.state.token), + } + + @app.get("/api/dashboard") + def dashboard( + target_version: str | None = None, + online: bool = False, + remote_tags: bool = False, + public_catalog: bool = True, + include_migrations: bool = False, + include_website: bool = False, + channel: str = "stable", + ) -> dict[str, object]: + snapshot = build_dashboard( + workspace_root=app.state.workspace_root, + target_version=target_version, + online=online, + check_remote_tags=online or remote_tags, + check_public_catalog=public_catalog, + include_migrations=include_migrations, + include_website=include_website, + channel=channel, + ) + return to_jsonable(snapshot) + + @app.get("/api/plan") + def plan( + target_version: str | None = None, + online: bool = False, + remote_tags: bool = False, + public_catalog: bool = True, + include_migrations: bool = False, + channel: str = "stable", + ) -> dict[str, object]: + snapshot = build_dashboard( + workspace_root=app.state.workspace_root, + target_version=target_version, + online=online, + check_remote_tags=online or remote_tags, + check_public_catalog=public_catalog, + include_migrations=include_migrations, + channel=channel, + ) + release_plan = build_release_plan(snapshot) + return to_jsonable(release_plan) + + @app.get("/api/release-intelligence") + def release_intelligence( + channel: str = "stable", + public_catalog: bool = True, + ) -> dict[str, object]: + intelligence = build_release_intelligence( + workspace_root=app.state.workspace_root, + channel=channel, + prefer_public=public_catalog, + ) + return to_jsonable(intelligence) + + @app.get("/api/selective-plan") + def selective_plan( + repos: str | None = None, + target_version: str | None = None, + repo_versions: str | None = None, + online: bool = False, + remote_tags: bool = False, + public_catalog: bool = True, + include_migrations: bool = False, + channel: str = "stable", + ) -> dict[str, object]: + snapshot = build_dashboard( + workspace_root=app.state.workspace_root, + target_version=target_version, + online=online, + check_remote_tags=online or remote_tags, + check_public_catalog=public_catalog, + include_migrations=include_migrations, + channel=channel, + ) + release_plan = build_selective_release_plan( + snapshot, + selected_repos=parse_csv(repos), + target_version=target_version, + repo_versions=parse_repo_versions(repo_versions), + channel=channel, + ) + return to_jsonable(release_plan) + + @app.get("/api/catalog-candidates") + def catalog_candidates() -> dict[str, object]: + return {"candidates": list_catalog_candidates()} + + @app.post("/api/catalog-candidates") + def catalog_candidate(request: CatalogCandidateRequest) -> dict[str, object]: + repo_versions = dict(request.repo_versions) + if not repo_versions and request.repos and request.target_version: + repo_versions = {repo: request.target_version for repo in request.repos} + if not repo_versions: + raise HTTPException(status_code=400, detail="repo_versions or repos with target_version is required") + signing_keys = tuple(request.signing_keys or default_signing_keys()) + if not signing_keys: + raise HTTPException(status_code=400, detail="No signing key supplied and default release key was not found") + candidate = build_selective_catalog_candidate( + repo_versions=repo_versions, + channel=request.channel, + workspace_root=app.state.workspace_root, + output_dir=request.output_dir, + base_catalog=request.base_catalog, + signing_keys=signing_keys, + public_base_url=request.public_base_url, + repository_base=request.repository_base, + expires_days=request.expires_days, + sequence=request.sequence, + check_public=request.check_public, + ) + return to_jsonable(candidate) + + @app.post("/api/catalog-candidates/publish") + def publish_candidate(request: PublishCandidateRequest) -> dict[str, object]: + if request.push and request.confirm != "PUSH": + raise HTTPException(status_code=400, detail="Push requires confirm=PUSH") + if (request.apply or request.commit or request.tag or request.build_web) and not request.push and request.confirm != "APPLY": + raise HTTPException(status_code=400, detail="Apply/commit/tag/build requires confirm=APPLY") + result = publish_catalog_candidate( + candidate_dir=request.candidate_dir, + workspace_root=app.state.workspace_root, + web_root=request.web_root, + channel=request.channel, + apply=request.apply, + build_web=request.build_web, + commit=request.commit, + tag=request.tag, + push=request.push, + remote=request.remote, + branch=request.branch, + tag_name=request.tag_name, + npm=request.npm, + allow_dirty_website=request.allow_dirty_website, + ) + return to_jsonable(result) + + @app.post("/api/repositories/push") + def repository_push(request: RepositoryPushRequest) -> dict[str, object]: + repos = tuple(dict.fromkeys(repo.strip() for repo in request.repos if repo.strip())) + if not repos: + raise HTTPException(status_code=400, detail="At least one repository must be selected") + if request.apply and request.confirm != "PUSH": + raise HTTPException(status_code=400, detail="Repository push requires confirm=PUSH") + result = push_repositories(repos=repos, workspace_root=app.state.workspace_root, remote=request.remote, apply=request.apply) + return to_jsonable(result) + + @app.post("/api/repositories/sync") + def repository_sync(request: RepositorySyncRequest) -> dict[str, object]: + repos = tuple(dict.fromkeys(repo.strip() for repo in request.repos if repo.strip())) + if not repos: + raise HTTPException(status_code=400, detail="At least one repository must be selected") + if request.apply and request.confirm != "SYNC": + raise HTTPException(status_code=400, detail="Repository sync requires confirm=SYNC") + result = sync_repositories(repos=repos, workspace_root=app.state.workspace_root, remote=request.remote, apply=request.apply) + return to_jsonable(result) + + @app.post("/api/repositories/prepare") + def repository_prepare(request: RepositoryPrepareRequest) -> dict[str, object]: + repos = tuple(dict.fromkeys(repo.strip() for repo in request.repos if repo.strip())) + if not repos: + raise HTTPException(status_code=400, detail="At least one repository must be selected") + if request.apply and request.confirm != "COMMIT": + raise HTTPException(status_code=400, detail="Repository prepare requires confirm=COMMIT") + result = prepare_repositories( + repos=repos, + repo_versions=request.repo_versions, + message=request.message, + workspace_root=app.state.workspace_root, + apply=request.apply, + ) + return to_jsonable(result) + + return app + + +def parse_csv(value: str | None) -> tuple[str, ...]: + if not value: + return () + return tuple(item.strip() for item in value.split(",") if item.strip()) + + +def parse_repo_versions(value: str | None) -> dict[str, str]: + if not value: + return {} + result: dict[str, str] = {} + for item in value.split(","): + if ":" not in item: + continue + repo, version = item.split(":", 1) + repo = repo.strip() + version = version.strip() + if repo and version: + result[repo] = version + return result + + +def default_signing_keys() -> tuple[str, ...]: + key_path = Path.home() / ".config" / "govoplan" / "release-keys" / "release-key-1.pem" + if not key_path.exists(): + return () + return (f"release-key-1={key_path}",) + + +def list_catalog_candidates() -> list[dict[str, object]]: + root = META_ROOT / "runtime" / "release-candidates" + if not root.exists(): + return [] + candidates: list[dict[str, object]] = [] + for path in sorted((item for item in root.iterdir() if item.is_dir()), reverse=True): + summary = path / "summary.json" + if summary.exists(): + try: + import json + + payload = json.loads(summary.read_text(encoding="utf-8")) + if isinstance(payload, dict): + payload.setdefault("candidate_dir", str(path)) + candidates.append(payload) + continue + except json.JSONDecodeError: + pass + candidates.append({"candidate_dir": str(path), "status": "unknown"}) + return candidates diff --git a/tools/release/webui/index.html b/tools/release/webui/index.html new file mode 100644 index 0000000..ca9dd08 --- /dev/null +++ b/tools/release/webui/index.html @@ -0,0 +1,1781 @@ + + + + + + GovOPlaN Release Console + + + +
+

GovOPlaN Release Console

+

Local release dashboard with guarded candidate generation and publication workflow actions.

+
+ +
+
+
+ + +
+ + + + + +
+
+
Loading...
+
+ +
+ +
+
+
+
+

Release Control

+ +
+
+
+
+ +
+
+

Published Modules

+ +
+
+ + + + + + + + + + + + + +
ModuleVersionRepoLocalInterfacesMigrationRefs
+
+
+ +
+
+

Compatibility

+ +
+
+ + + + + + + + + + + +
ConsumerInterfaceRangeStatusProvider
+
+
+ +
+
+

Repositories

+ +
+
+ + + + + + +
+
+ + + + + + + + + + + + + + + +
ReleaseRepositoryTypeCurrentTargetTarget tagBranch / UpstreamGitNotes
+
+
+
+ +
+
+
+

Release Workflow

+ +
+
+
+ +
+
+

1. Git Sync

+ +
+
+

Sync selected fetches remote refs and tags only. It does not merge, rebase, commit, tag, or push.

+
+
+ + +
+
+ + +
+
+
+ + + + +
+
+
+
+ +
+
+

2. Prepare Changes

+ +
+
+

Prepare commits selected dirty worktrees only. It does not tag or push.

+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+
+ +
+
+

3. Release Tag Plan

+ +
+
+
+ +
+
+

4. Signed Release

+ +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + + + +
+
+
+
+ +
+
+

Catalog State

+ +
+
+
+ +
+
+

5. Install Catalog

+ planned +
+
+

Future work: signed module archives and an installer/update catalog for deployments.

+
+
+
+
+
+ + + + + + diff --git a/tools/repo/generate-repository-index.py b/tools/repo/generate-repository-index.py new file mode 100644 index 0000000..ec7e340 --- /dev/null +++ b/tools/repo/generate-repository-index.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CATEGORIES = ("system", "module", "connector", "website") + + +def main() -> int: + manifest = json.loads((ROOT / "repositories.json").read_text(encoding="utf-8")) + lines = [ + "# GovOPlaN Repository Index", + "", + "Generated from `repositories.json`. Use that JSON file as the machine-readable source of truth; this page is the human-readable link index.", + "", + ] + for category in CATEGORIES: + entries = [entry for entry in manifest["repositories"] if entry["category"] == category] + if not entries: + continue + lines.extend([ + f"## {category.title()}", + "", + "| Repository | Subtype | Local path | Gitea |", + "| --- | --- | --- | --- |", + ]) + for entry in entries: + name = entry["name"] + subtype = entry.get("subtype", "") + path = entry["path"] + url = gitea_url(entry["remote"]) + lines.append(f"| `{name}` | `{subtype}` | `../{path}` | [{name}]({url}) |") + lines.append("") + (ROOT / "docs" / "REPOSITORY_INDEX.md").write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + return 0 + + +def gitea_url(remote: str) -> str: + if remote.startswith("git@git.add-ideas.de:"): + return remote.replace("git@git.add-ideas.de:", "https://git.add-ideas.de/", 1).removesuffix(".git") + return remote + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/repo/sync-python-environment.py b/tools/repo/sync-python-environment.py new file mode 100644 index 0000000..5f6c285 --- /dev/null +++ b/tools/repo/sync-python-environment.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +"""Synchronize a GovOPlaN Python environment when package metadata changes.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +from typing import Any + +META_ROOT = Path(__file__).resolve().parents[2] +STAMP_VERSION = 1 + + +@dataclass(frozen=True, slots=True) +class RequirementEntry: + line: str + install_args: tuple[str, ...] + is_local: bool + key: str + target_path: str | None = None + pyproject: str | None = None + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "line": self.line, + "install_args": list(self.install_args), + "is_local": self.is_local, + "key": self.key, + } + if self.target_path: + payload["target_path"] = self.target_path + if self.pyproject: + payload["pyproject"] = self.pyproject + return payload + + +@dataclass(frozen=True, slots=True) +class InstallPlan: + mode: str + reason: str + commands: tuple[tuple[str, ...], ...] + warnings: tuple[str, ...] = () + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--requirements", + type=Path, + default=META_ROOT / "requirements-dev.txt", + help="Requirements file to install. Defaults to requirements-dev.txt.", + ) + parser.add_argument( + "--python", + default=sys.executable, + help="Python executable to use for pip. Defaults to the current interpreter.", + ) + parser.add_argument( + "--stamp", + type=Path, + default=None, + help="Stamp file used to remember the last synchronized metadata digest.", + ) + parser.add_argument("--force", action="store_true", help="Run pip even when the environment stamp is current.") + parser.add_argument("--check", action="store_true", help="Exit 1 if synchronization would be required.") + parser.add_argument("--dry-run", action="store_true", help="Print what would run without changing the environment.") + parser.add_argument("--upgrade-pip", action="store_true", help="Upgrade pip before installing requirements.") + args = parser.parse_args() + + requirements = args.requirements.expanduser().resolve() + if not requirements.exists(): + print(f"Requirements file not found: {requirements}", file=sys.stderr) + return 2 + + python = normalize_python_command(args.python) + stamp = (args.stamp or default_stamp_path(python, requirements)).expanduser().resolve() + local_requirements = local_requirement_entries(requirements) + fingerprint = build_fingerprint(requirements=requirements, python=python, local_requirements=local_requirements) + previous = load_stamp(stamp) + current = previous.get("digest") == fingerprint["digest"] + + if current and not args.force: + payload = stamp_payload( + requirements=requirements, + python=python, + fingerprint=fingerprint, + entries=parse_requirement_entries(requirements), + ) + if stamp_needs_refresh(previous, payload): + if args.dry_run: + print(f"Python environment is current for {requirements.name}. Stamp metadata would be refreshed.") + print("No pip command required; the environment stamp would be refreshed.") + return 0 + if args.check: + print(f"Python environment is current for {requirements.name}.") + return 0 + write_stamp(stamp, payload) + print(f"Python environment is current for {requirements.name}. Stamp metadata refreshed.") + return 0 + print(f"Python environment is current for {requirements.name}.") + return 0 + + print(f"Python environment metadata is stale for {requirements.name}.") + print(f"Tracked inputs: {len(fingerprint['inputs'])}") + + if args.check: + return 1 + + plan = build_install_plan( + previous=previous, + fingerprint=fingerprint, + requirements=requirements, + python=python, + local_requirements=local_requirements, + force=args.force, + ) + + print(f"{plan.mode}: {plan.reason}") + for warning in plan.warnings: + print(f"warning: {warning}", file=sys.stderr) + + commands: list[tuple[str, ...]] = [] + if args.upgrade_pip: + commands.append((python, "-m", "pip", "install", "--upgrade", "pip")) + commands.extend(plan.commands) + + if args.dry_run: + for command in commands: + print("+ " + format_command(command)) + if not commands: + print("No pip command required; the environment stamp would be refreshed.") + return 0 + + for command in commands: + subprocess.run(command, check=True) + + write_stamp( + stamp, + stamp_payload( + requirements=requirements, + python=python, + fingerprint=fingerprint, + entries=parse_requirement_entries(requirements), + ), + ) + print(f"Python environment synchronized. Stamp written to {stamp}.") + return 0 + + +def default_stamp_path(python: str, requirements: Path) -> Path: + executable = Path(python) + if executable.name.startswith("python") and executable.parent.name == "bin": + return executable.parent.parent / f".govoplan-sync-{requirements.stem}.json" + return META_ROOT / "runtime" / f".govoplan-sync-{requirements.stem}.json" + + +def normalize_python_command(value: str) -> str: + if os.sep not in value: + return value + path = Path(value).expanduser() + if not path.is_absolute(): + path = Path.cwd() / path + return str(path) + + +def stamp_payload( + *, + requirements: Path, + python: str, + fingerprint: dict[str, Any], + entries: tuple[RequirementEntry, ...], +) -> dict[str, object]: + return { + "version": STAMP_VERSION, + "digest": fingerprint["digest"], + "requirements": str(requirements), + "python": python, + "inputs": fingerprint["inputs"], + "requirements_entries": [entry.as_dict() for entry in entries], + } + + +def stamp_needs_refresh(previous: dict[str, Any], payload: dict[str, object]) -> bool: + return any(previous.get(key) != value for key, value in payload.items()) + + +def build_fingerprint(*, requirements: Path, python: str, local_requirements: tuple[RequirementEntry, ...]) -> dict[str, Any]: + hasher = hashlib.sha256() + inputs: list[dict[str, str]] = [] + + add_text(hasher, "stamp-version", str(STAMP_VERSION)) + add_text(hasher, "python", python) + add_file(hasher, inputs, requirements) + + for project in local_pyprojects(local_requirements): + add_file(hasher, inputs, project) + + return {"digest": hasher.hexdigest(), "inputs": inputs} + + +def build_install_plan( + *, + previous: dict[str, Any], + fingerprint: dict[str, Any], + requirements: Path, + python: str, + local_requirements: tuple[RequirementEntry, ...], + force: bool, +) -> InstallPlan: + full_command = (python, "-m", "pip", "install", "-r", str(requirements)) + if force: + return InstallPlan("Full Python environment sync", "--force was requested.", (full_command,)) + if previous.get("version") != STAMP_VERSION: + return InstallPlan("Full Python environment sync", "the environment stamp is missing or uses an older format.", (full_command,)) + if previous.get("python") != python: + return InstallPlan("Full Python environment sync", "the Python executable changed.", (full_command,)) + + previous_inputs = inputs_by_path(previous.get("inputs")) + current_inputs = inputs_by_path(fingerprint.get("inputs")) + if not previous_inputs: + return InstallPlan("Full Python environment sync", "there is no previous input fingerprint.", (full_command,)) + + requirements_key = str(requirements) + changed_paths = { + path + for path, digest in current_inputs.items() + if previous_inputs.get(path) != digest + } + removed_paths = set(previous_inputs) - set(current_inputs) + stale_requirements = requirements_key in changed_paths or requirements_key in removed_paths + + commands: list[tuple[str, ...]] = [] + warnings: list[str] = [] + + if stale_requirements: + delta = requirement_delta_plan(previous.get("requirements_entries"), parse_requirement_entries(requirements)) + if delta is None: + return InstallPlan( + "Full Python environment sync", + "requirements changed in a way that needs the full resolver.", + (full_command,), + ) + for install_args in delta.commands: + commands.append((python, "-m", "pip", "install", *install_args)) + warnings.extend(delta.warnings) + changed_paths.discard(requirements_key) + removed_paths.discard(requirements_key) + + local_by_pyproject = { + entry.pyproject: entry + for entry in local_requirements + if entry.pyproject + } + for path in sorted(changed_paths): + entry = local_by_pyproject.get(path) + if entry is None: + return InstallPlan( + "Full Python environment sync", + f"tracked input changed outside a known local project: {path}", + (full_command,), + ) + commands.append((python, "-m", "pip", "install", *entry.install_args)) + + if removed_paths: + return InstallPlan( + "Full Python environment sync", + "tracked input files disappeared; using the full requirements installer.", + (full_command,), + ) + + deduped_commands = tuple(dict.fromkeys(commands)) + if deduped_commands: + return InstallPlan( + "Selective Python environment sync", + f"installing {len(deduped_commands)} stale local requirement(s).", + deduped_commands, + tuple(warnings), + ) + return InstallPlan( + "Stamp-only Python environment sync", + "requirements metadata changed without a pip-installable requirement change.", + (), + tuple(warnings), + ) + + +def requirement_delta_plan(previous_entries: object, current_entries: tuple[RequirementEntry, ...]) -> InstallPlan | None: + parsed_previous = previous_requirement_entries(previous_entries) + if parsed_previous is None: + return None + + previous_by_key = {entry.key: entry for entry in parsed_previous} + current_by_key = {entry.key: entry for entry in current_entries} + changed_current = [ + entry + for key, entry in current_by_key.items() + if previous_by_key.get(key) is None or previous_by_key[key].install_args != entry.install_args or previous_by_key[key].line != entry.line + ] + removed_keys = set(previous_by_key) - set(current_by_key) + if any(not entry.is_local or not entry.pyproject for entry in changed_current): + return None + + warnings = [] + if removed_keys: + warnings.append("Removed requirements are not uninstalled automatically; pip install -r had the same limitation.") + commands = tuple((entry.install_args) for entry in changed_current) + return InstallPlan( + "Requirement delta", + "requirements file changed only in local editable requirements.", + commands, + tuple(warnings), + ) + + +def previous_requirement_entries(value: object) -> tuple[RequirementEntry, ...] | None: + if not isinstance(value, list): + return None + entries: list[RequirementEntry] = [] + for item in value: + if not isinstance(item, dict): + return None + raw_install_args = item.get("install_args") + if not isinstance(raw_install_args, list) or not all(isinstance(arg, str) for arg in raw_install_args): + return None + line = item.get("line") + key = item.get("key") + if not isinstance(line, str) or not isinstance(key, str): + return None + entries.append( + RequirementEntry( + line=line, + install_args=tuple(raw_install_args), + is_local=bool(item.get("is_local")), + key=key, + target_path=item.get("target_path") if isinstance(item.get("target_path"), str) else None, + pyproject=item.get("pyproject") if isinstance(item.get("pyproject"), str) else None, + ) + ) + return tuple(entries) + + +def inputs_by_path(value: object) -> dict[str, str]: + if not isinstance(value, list): + return {} + inputs: dict[str, str] = {} + for item in value: + if not isinstance(item, dict): + continue + path = item.get("path") + digest = item.get("sha256") + if isinstance(path, str) and isinstance(digest, str): + inputs[path] = digest + return inputs + + +def local_requirement_entries(requirements: Path) -> tuple[RequirementEntry, ...]: + return tuple(entry for entry in parse_requirement_entries(requirements) if entry.is_local and entry.pyproject) + + +def parse_requirement_entries(requirements: Path) -> tuple[RequirementEntry, ...]: + entries: list[RequirementEntry] = [] + for raw_line in requirements.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or line.startswith(("-r ", "--requirement")): + continue + editable = editable_target(line) + target = editable or local_path_target(line) + if target is None: + entries.append(RequirementEntry(line=line, install_args=(line,), is_local=False, key=f"requirement:{line}")) + continue + path = resolve_requirement_path(requirements.parent, target) + if path is None: + entries.append(RequirementEntry(line=line, install_args=(line,), is_local=False, key=f"requirement:{line}")) + continue + pyproject = path / "pyproject.toml" + resolved_pyproject = str(pyproject.resolve()) if pyproject.exists() else None + install_target = absolute_local_target(requirements.parent, target) + install_args = ("-e", install_target) if editable is not None else (install_target,) + key = f"local:{resolved_pyproject or path.resolve()}" + entries.append( + RequirementEntry( + line=line, + install_args=install_args, + is_local=True, + key=key, + target_path=str(path.resolve()), + pyproject=resolved_pyproject, + ) + ) + return tuple(entries) + + +def local_pyprojects(local_requirements: tuple[RequirementEntry, ...]) -> tuple[Path, ...]: + projects = {Path(entry.pyproject) for entry in local_requirements if entry.pyproject} + return tuple(sorted(projects)) + + +def editable_target(line: str) -> str | None: + if line.startswith("-e "): + return line[3:].strip() + if line.startswith("--editable "): + return line.removeprefix("--editable ").strip() + if line.startswith("--editable="): + return line.removeprefix("--editable=").strip() + return None + + +def local_path_target(line: str) -> str | None: + if line.startswith(("./", "../", "/")): + return line + return None + + +def resolve_requirement_path(base: Path, target: str) -> Path | None: + cleaned, _suffix = split_local_target(target) + if "://" in cleaned or cleaned.startswith("git+"): + return None + if not cleaned: + return None + return (base / cleaned).resolve() + + +def absolute_local_target(base: Path, target: str) -> str: + path_part, suffix = split_local_target(target) + return f"{(base / path_part).resolve()}{suffix}" + + +def split_local_target(target: str) -> tuple[str, str]: + cleaned = target.split("#", 1)[0].strip() + fragment = target[len(cleaned) :].strip() + extras = "" + if "[" in cleaned: + cleaned, extra_part = cleaned.split("[", 1) + extras = f"[{extra_part}" + return cleaned.strip(), f"{extras}{fragment}" + + +def add_file(hasher: hashlib._Hash, inputs: list[dict[str, str]], path: Path) -> None: + data = path.read_bytes() + digest = hashlib.sha256(data).hexdigest() + add_text(hasher, str(path), digest) + inputs.append({"path": str(path), "sha256": digest}) + + +def add_text(hasher: hashlib._Hash, key: str, value: str) -> None: + hasher.update(key.encode("utf-8")) + hasher.update(b"\0") + hasher.update(value.encode("utf-8")) + hasher.update(b"\0") + + +def load_stamp(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, dict) else {} + + +def write_stamp(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def format_command(command: tuple[str, ...]) -> str: + return " ".join(shlex.quote(item) for item in command) + + +if __name__ == "__main__": + raise SystemExit(main())