Compare commits
11 Commits
17f8036bdc
...
99e255cf81
| Author | SHA1 | Date | |
|---|---|---|---|
| 99e255cf81 | |||
| d9819c4aad | |||
| 78b601a2e5 | |||
| a18c7ea040 | |||
| b193d4555f | |||
| a81dc21e90 | |||
| 676030b993 | |||
| a15a74c54c | |||
| c34892f6ea | |||
| 397c87a012 | |||
| f1b07354a0 |
16
.env.example
16
.env.example
@@ -15,7 +15,23 @@ REDIS_URL=redis://127.0.0.1:6379/0
|
||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||
|
||||
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=false
|
||||
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||
GOVOPLAN_HTTP_HSTS_SECONDS=31536000
|
||||
|
||||
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||
|
||||
CORS_ORIGINS=https://govoplan.example.org
|
||||
GOVOPLAN_TRUSTED_HOSTS=govoplan.example.org
|
||||
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||
AUTH_COOKIE_SECURE=true
|
||||
AUTH_COOKIE_SAMESITE=lax
|
||||
AUTH_COOKIE_DOMAIN=
|
||||
|
||||
24
README.md
24
README.md
@@ -78,6 +78,27 @@ Run the static cross-repository module contract check:
|
||||
./tools/checks/check-contracts.sh
|
||||
```
|
||||
|
||||
Require backend, manifest, frontend, lockfile, and release-composition versions
|
||||
to agree before a release:
|
||||
|
||||
```sh
|
||||
./.venv/bin/python tools/checks/check-version-alignment.py --release-composition
|
||||
```
|
||||
|
||||
Generate the CycloneDX dependency inventory from a resolved release environment:
|
||||
|
||||
```sh
|
||||
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
||||
```
|
||||
|
||||
For reproducible release artifacts, set `SOURCE_DATE_EPOCH` to the release
|
||||
commit timestamp (or pass an explicit timezone-qualified `--timestamp`):
|
||||
|
||||
```sh
|
||||
SOURCE_DATE_EPOCH="$(git -C ../govoplan-core show -s --format=%ct HEAD)" \
|
||||
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
||||
```
|
||||
|
||||
Run the consolidated focused verification suite:
|
||||
|
||||
```sh
|
||||
@@ -128,6 +149,9 @@ Frontend layout principles for module pages are documented in
|
||||
The cross-product destination, stakeholder visions, configuration archetypes,
|
||||
connected outcome stories, and capability horizons are documented in
|
||||
the [Connected Governance Platform Roadmap](docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md).
|
||||
The administrator journey from Core-only bootstrap through online module
|
||||
installation, scale-out, and reversible environment promotion is defined in
|
||||
[System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md).
|
||||
The first Campaign-centric capability and infrastructure fit assessment is in
|
||||
`docs/CAPABILITY_AND_INFRASTRUCTURE_FIT.md`.
|
||||
|
||||
|
||||
@@ -19,8 +19,24 @@ CELERY_ENABLED=true
|
||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||
|
||||
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=true
|
||||
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||
GOVOPLAN_HTTP_HSTS_SECONDS=0
|
||||
|
||||
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||
GOVOPLAN_TRUSTED_HOSTS=127.0.0.1,localhost,testserver
|
||||
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||
AUTH_COOKIE_SECURE=false
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||
|
||||
@@ -253,6 +253,12 @@ Operational access is distinct from permission to read all business content.
|
||||
Diagnostics minimize personal data and link to governed evidence when deeper
|
||||
inspection is authorized.
|
||||
|
||||
The complete installation and lifecycle journey is specified in the
|
||||
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
|
||||
one-command Core-only bootstrap, signed online module installation and updates,
|
||||
stateless scale-out, versioned configuration transfer, undo, and reproducible
|
||||
environment-promotion recipes.
|
||||
|
||||
### Integration owner and enterprise architect
|
||||
|
||||
> As an integration owner, I can inventory external systems, declare direction
|
||||
@@ -871,23 +877,27 @@ checkouts.
|
||||
|
||||
Priorities:
|
||||
|
||||
1. Pin and publish a compatible Core/WebUI/module composition and first
|
||||
1. Deliver the first slices of the
|
||||
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
|
||||
a verified Core-only distribution, first-run control plane, read-only online
|
||||
module directory, and durable plan/confirm/install progress.
|
||||
2. Pin and publish a compatible Core/WebUI/module composition and first
|
||||
reference configuration package.
|
||||
2. Complete database, files, configuration, secrets, and audit backup/restore
|
||||
3. Complete database, files, configuration, secrets, and audit backup/restore
|
||||
drills with measured recovery objectives.
|
||||
3. Prove the providers required by the selected reference configurations,
|
||||
4. Prove the providers required by the selected reference configurations,
|
||||
beginning with SMTP/IMAP, CalDAV, and the relevant broker/worker and file or
|
||||
directory adapters, including reconciliation behavior.
|
||||
4. Add external monitoring, alerting, audit export/retention, deployment
|
||||
5. Add external monitoring, alerting, audit export/retention, deployment
|
||||
hardening, and incident/runbook evidence.
|
||||
5. Apply the interface pattern language first to Campaign, scheduling, and
|
||||
6. Apply the interface pattern language first to Campaign, scheduling, and
|
||||
admin/configuration. Existing central components are mandatory; any custom
|
||||
control needs prior product-owner authorization and a narrow recorded scope.
|
||||
Extend central contracts only after a repeated need is demonstrated.
|
||||
6. Align executable WebUI contributions with manifest/configured-system
|
||||
7. Align executable WebUI contributions with manifest/configured-system
|
||||
metadata and make focused views manually/default selectable without waiting
|
||||
for Workflow.
|
||||
7. Refresh the capability/infrastructure assessment and distinguish code-tested,
|
||||
8. Refresh the capability/infrastructure assessment and distinguish code-tested,
|
||||
target-tested, package-integrated, pilot-approved, and production-approved
|
||||
evidence.
|
||||
|
||||
|
||||
@@ -151,6 +151,24 @@ forcing every repository to the same tag, but channel publication must preserve
|
||||
the unchanged package versions, validate interface compatibility, sign the
|
||||
updated catalog, and keep the published keyring healthy.
|
||||
|
||||
Release integration also enforces repository and composition version alignment
|
||||
and generates a CycloneDX SBOM from the resolved Python environment and the
|
||||
release WebUI lockfile. Catalog publication should attach that immutable SBOM
|
||||
and its digest to the corresponding Core/composition release.
|
||||
|
||||
Addresses and Notifications have current WebUI source contributions but are
|
||||
not part of the pinned v0.1.8 WebUI release composition because their v0.1.8
|
||||
tags predate those packages. They re-enter the release composition only through
|
||||
new, immutable module tags whose backend, manifest, frontend, and lock metadata
|
||||
pass the alignment gate.
|
||||
|
||||
Candidate publication verifies signatures against the already published
|
||||
keyring, not against keys supplied only by the candidate. A changed keyring
|
||||
must have its canonical SHA-256 embedded in the signed catalog. The public
|
||||
module-directory files are regenerated from that verified catalog and keyring
|
||||
at publication time; candidate-supplied directory files are never copied as
|
||||
authoritative provenance.
|
||||
|
||||
## Published Module Directory
|
||||
|
||||
The target release repository is an online, browsable module directory. The
|
||||
|
||||
@@ -34,16 +34,28 @@ tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||
```
|
||||
|
||||
Reports are written to `audit-reports/`, which is intentionally ignored by git.
|
||||
Each run records tool versions, report checksums, and start/end repository
|
||||
revision plus worktree fingerprints. A repository change during scanning makes
|
||||
the run fail so a mixed code snapshot cannot be reported as a valid audit.
|
||||
Step exit codes are recorded separately from findings: report-only mode may
|
||||
accept findings, but scanner execution errors and malformed JSON/SARIF reports
|
||||
always fail the run. The manifest lists the expected, present, and missing
|
||||
reports for that invocation. Validation and checksums use that explicit set, so
|
||||
reusing a report directory cannot make stale output look like part of a new run.
|
||||
|
||||
The wrapper tags the toolbox image by a fingerprint of the Dockerfile and
|
||||
`requirements-audit.txt`. If those inputs have not changed, subsequent runs reuse
|
||||
the existing local image instead of reinstalling all tools. The stable alias is
|
||||
`govoplan/security-audit:local` unless `SECURITY_AUDIT_IMAGE` is set.
|
||||
The wrapper tags the toolbox image by a fingerprint of the Dockerfile,
|
||||
`requirements-audit.txt`, and the Semgrep smoke-test inputs. If those inputs have
|
||||
not changed, subsequent runs reuse the existing local image instead of
|
||||
reinstalling all tools. The stable alias is `govoplan/security-audit:local`
|
||||
unless `SECURITY_AUDIT_IMAGE` is set.
|
||||
|
||||
Semgrep is installed separately in the toolbox image because current Semgrep
|
||||
packages pin Click to an affected `8.1.x` line. The image upgrades Click after
|
||||
Semgrep installation and fails during build if Semgrep no longer starts with the
|
||||
patched Click version.
|
||||
packages pin affected Click and MCP versions. The image upgrades both after
|
||||
Semgrep installation, runs an actual local-rule scan, and audits the final
|
||||
toolbox Python environment during the image build. The compatibility releases
|
||||
are pinned exactly to keep the tested override reproducible. Remove either
|
||||
compatibility override only after Semgrep's own dependency bounds include a
|
||||
fixed version.
|
||||
|
||||
Force a cached rebuild:
|
||||
|
||||
@@ -70,6 +82,10 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build
|
||||
- `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit.
|
||||
- `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon.
|
||||
|
||||
Semgrep and Trivy are invoked with finding-sensitive exit codes. Their exit 1
|
||||
is therefore a finding under the wrapper contract; higher exit codes, missing
|
||||
output, invalid JSON/SARIF, and scanner error payloads are execution failures.
|
||||
|
||||
Bandit and Ruff security reports are split by source kind. Production code under
|
||||
`src/` is written to `bandit.json` and `ruff-security.json` and controls strict
|
||||
mode. Test code under `tests/` is still scanned for visibility, but its findings
|
||||
@@ -153,7 +169,10 @@ cd /mnt/DATA/git/govoplan
|
||||
python -m venv .venv
|
||||
./.venv/bin/python -m pip install -r requirements-audit.txt
|
||||
./.venv/bin/python -m pip install 'semgrep>=1.140,<2'
|
||||
./.venv/bin/python -m pip install --upgrade --no-deps 'click>=8.3.3'
|
||||
./.venv/bin/python -m pip install --upgrade --no-deps 'click==8.3.3'
|
||||
./.venv/bin/python -m pip install --upgrade --no-deps 'mcp==1.28.1'
|
||||
./.venv/bin/semgrep scan --metrics=off --config tools/checks/security-audit/semgrep-govoplan.yml tools/checks/check-version-alignment.py
|
||||
./.venv/bin/pip-audit --progress-spinner off
|
||||
```
|
||||
|
||||
Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`)
|
||||
|
||||
169
docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md
Normal file
169
docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md
Normal file
@@ -0,0 +1,169 @@
|
||||
# System Administrator Lifecycle User Story
|
||||
|
||||
## Story
|
||||
|
||||
> As a system administrator, I can execute one shell command that downloads a
|
||||
> verified GovOPlaN distribution and starts a completely configured Core control
|
||||
> plane without optional modules. In the WebUI I can browse compatible signed
|
||||
> module releases, select the modules for this installation, and follow every
|
||||
> download, validation, migration, installation, activation, and health-check
|
||||
> step. When an update is available, I can review its impact and confirm it.
|
||||
>
|
||||
> I can add API or worker replicas on containers or other servers without
|
||||
> copying mutable local state. I can operate development, test, and production
|
||||
> systems, transfer a versioned configuration through the WebUI, undo an applied
|
||||
> configuration version, test an update in one environment, and then apply the
|
||||
> same immutable update recipe to another environment.
|
||||
|
||||
This is a product-level story owned by the GovOPlaN platform rather than by an
|
||||
individual domain module. It joins installation, module lifecycle, operations,
|
||||
configuration packages, and release provenance into one administrator journey.
|
||||
The canonical backlog item is
|
||||
[GovOPlaN #13](https://git.add-ideas.de/add-ideas/govoplan/issues/13).
|
||||
|
||||
## Terms
|
||||
|
||||
- **Core control plane:** the smallest bootable distribution: Core API, Core
|
||||
WebUI, PostgreSQL, Redis, installer worker, migration runner, and durable
|
||||
storage configuration. No optional GovOPlaN module package is installed.
|
||||
- **Bootstrap administrator:** a single-use, time-limited installation identity
|
||||
that may access only first-run and module-lifecycle functions. It is retired
|
||||
when the selected identity/access configuration becomes healthy.
|
||||
- **Module release:** an immutable, signed package plus manifest, compatibility
|
||||
contract, migrations, WebUI contribution, checksums, and SBOM references.
|
||||
- **Configuration revision:** an immutable, schema-versioned export of
|
||||
non-secret system/module settings, policies, compositions, and secret
|
||||
references. Secret values are never exported.
|
||||
- **Update recipe:** the reviewed, immutable plan containing exact Core/module
|
||||
versions, configuration revision, migration order, preflight results,
|
||||
maintenance/drain requirements, health checks, and permitted rollback or
|
||||
forward-recovery actions.
|
||||
|
||||
## Acceptance journeys
|
||||
|
||||
### One-command first installation
|
||||
|
||||
1. The administrator runs one documented command on a supported host.
|
||||
2. The bootstrapper verifies a signed distribution manifest before executing or
|
||||
starting downloaded artifacts.
|
||||
3. It checks container/runtime, ports, storage, entropy, memory, architecture,
|
||||
and connectivity requirements; generates deployment-local secrets with
|
||||
restrictive permissions; and never prints them.
|
||||
4. It starts PostgreSQL, Redis, Core API/WebUI, one installer worker, and the
|
||||
migration runner. Readiness does not pass until migrations and durable
|
||||
dependencies are healthy.
|
||||
5. It prints the local URL and one-time bootstrap credential. Re-running the
|
||||
command is idempotent and shows or repairs the existing installation rather
|
||||
than creating another identity or database.
|
||||
6. No optional module is installed or enabled at this point.
|
||||
|
||||
### Module selection, installation, and update
|
||||
|
||||
1. Core reads an approved signed catalog and trusted keyring through the shared
|
||||
outbound-network policy.
|
||||
2. The WebUI shows available, installed, compatible, blocked, withdrawn, and
|
||||
update-available releases with their channel, provenance, contracts,
|
||||
migrations, permissions, configuration requirements, and release notes.
|
||||
3. Selecting modules produces a dependency/compatibility plan before any
|
||||
mutation. The administrator can amend the selection or confirm the plan.
|
||||
4. Installation executes durably in a worker. The UI receives persisted step
|
||||
state and can reconnect without losing progress.
|
||||
5. Package signature/checksum, version alignment, contract closure, migration
|
||||
graph, configuration schema, and health checks are mandatory gates.
|
||||
6. A failed or interrupted operation reaches a visible retry, forward-recovery,
|
||||
rollback, or manual-intervention state. It never reports success merely
|
||||
because the initiating request returned.
|
||||
7. An available update follows the same preview and confirmation path. Exact
|
||||
repeated requests are idempotent.
|
||||
|
||||
### Horizontal scaling
|
||||
|
||||
1. API, WebUI, installer, scheduler, and ordinary worker roles are stateless
|
||||
with respect to local container disks. Durable state uses PostgreSQL, Redis,
|
||||
and configured shared file/object storage.
|
||||
2. A documented command can add API or queue-specific worker replicas. The
|
||||
default Compose profile supports local scale-out; an orchestrator profile
|
||||
supplies equivalent health/readiness probes and rolling replacement.
|
||||
3. Only one migration or module-lifecycle mutation may own the deployment lock,
|
||||
while any healthy replica can serve read and normal domain traffic.
|
||||
4. Workers announce identity, queues, software composition, heartbeat, and
|
||||
drain state. Operators can see skew and safely retire a replica.
|
||||
5. Sessions, throttling, idempotency, jobs, installer progress, and scheduled
|
||||
work remain correct when requests move between replicas.
|
||||
|
||||
### Development, test, and production promotion
|
||||
|
||||
1. The administrator exports a configuration revision from development through
|
||||
the WebUI. The package is versioned, checksummed, attributable, and contains
|
||||
secret references or required-secret declarations, never secret values.
|
||||
2. Test imports the package into a preview area. Core reports environment-bound
|
||||
values, missing capabilities/secrets, compatibility changes, and the exact
|
||||
apply plan.
|
||||
3. Applying creates a new revision; the previous effective revision remains
|
||||
addressable. Undo is a new audited revision that restores the earlier
|
||||
configuration where contracts permit it.
|
||||
4. After tests and health checks pass, Core emits an update recipe with exact
|
||||
immutable release and configuration identifiers.
|
||||
5. Production validates the recipe against its own environment, requires a new
|
||||
confirmation, and executes the same ordered plan. Environment-specific
|
||||
secret bindings and endpoints remain local.
|
||||
6. Promotion never copies tenant/business data implicitly. Data migration,
|
||||
anonymized fixtures, and backup/restore are separate explicit operations.
|
||||
|
||||
## Safety and governance requirements
|
||||
|
||||
- Catalogs, artifacts, recipes, and configuration revisions have signatures,
|
||||
checksums, provenance, expiry/revocation semantics, and audit evidence.
|
||||
- The installer has a dedicated narrow authority; normal domain permissions do
|
||||
not imply host/package-management access.
|
||||
- The UI distinguishes reversible configuration rollback from database or
|
||||
package migrations that require forward recovery.
|
||||
- Cluster-wide mutations use fencing/leases so an expired worker cannot later
|
||||
commit a stale result.
|
||||
- Update plans declare availability impact, worker draining, backup/restore
|
||||
prerequisites, database compatibility windows, and post-change probes.
|
||||
- Deployment profiles set secure headers, trusted proxies/hosts, body limits,
|
||||
outbound-network policy, storage, TLS/cookie posture, and observability.
|
||||
- Release publication requires aligned backend/frontend/manifest versions and
|
||||
machine-readable dependency/SBOM provenance.
|
||||
|
||||
## Implementation slices
|
||||
|
||||
1. **Reproducible Core-only distribution.** Publish pinned multi-architecture
|
||||
images, signed distribution manifest, Core-only Compose profile, bootstrap
|
||||
preflight, generated secrets, readiness, and idempotent rerun/repair.
|
||||
2. **First-run control plane.** Add the restricted bootstrap administrator,
|
||||
one-time enrollment, initial catalog/keyring configuration, and retirement
|
||||
after durable administrator access is established.
|
||||
3. **Read-only online module directory.** Move the existing catalog and module
|
||||
directory contracts into the installed Core WebUI with compatibility,
|
||||
provenance, release-note, and update-state presentation.
|
||||
4. **Durable module plan and install.** Reuse the existing installer queue,
|
||||
locks, signed-package validator, rollback drill, and run evidence behind a
|
||||
plan/confirm/progress UI. Add initial catalog-entry synthesis and artifact
|
||||
acquisition where the current release console still assumes local sources.
|
||||
5. **Safe module update.** Add drain/maintenance coordination, backup gate,
|
||||
migration compatibility window, reconnectable progress, health verification,
|
||||
retry/recovery, and update notification.
|
||||
6. **Stateless replica profile.** Remove remaining local-runtime assumptions,
|
||||
expose role-specific commands/images, implement worker registration/drain,
|
||||
and prove multiple API and worker replicas against shared dependencies.
|
||||
7. **Configuration revision model.** Define provider export/import schemas,
|
||||
canonical serialization, secret references, validation/diff, immutable
|
||||
revision storage, audit, apply, and undo-as-new-revision.
|
||||
8. **Environment promotion and recipes.** Add source/target fingerprints,
|
||||
preview, environment bindings, acceptance evidence, exact recipe generation,
|
||||
signed transfer, and independently confirmed application.
|
||||
9. **Operational proof.** Exercise interrupted installs, stale locks, unavailable
|
||||
catalogs, revoked keys, failed migrations, replica loss, configuration undo,
|
||||
and development-to-test-to-production promotion in release CI and target
|
||||
drills.
|
||||
|
||||
## Explicit non-goals for the first distribution slice
|
||||
|
||||
- Shipping optional modules in the Core image.
|
||||
- Exporting secrets or production business data with configuration.
|
||||
- Pretending every schema migration can be reversed automatically.
|
||||
- Building a proprietary orchestrator instead of supporting Compose and a
|
||||
standard cluster scheduler through the same role/readiness contracts.
|
||||
- Allowing the browser process to execute arbitrary shell commands.
|
||||
@@ -1,6 +1,5 @@
|
||||
# Whole-product release install from tagged module repositories.
|
||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when
|
||||
# cutting a release.
|
||||
# Whole-product release install from immutable, independently versioned module tags.
|
||||
# Only add a module after its referenced tag has been published.
|
||||
../govoplan-core[server]
|
||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.8
|
||||
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.8
|
||||
@@ -11,14 +10,9 @@ govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.
|
||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.8
|
||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.8
|
||||
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.8
|
||||
govoplan-addresses @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-addresses.git@v0.1.8
|
||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.8
|
||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.8
|
||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.8
|
||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.8
|
||||
govoplan-poll @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-poll.git@v0.1.8
|
||||
govoplan-scheduling @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-scheduling.git@v0.1.8
|
||||
govoplan-notifications @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-notifications.git@v0.1.8
|
||||
govoplan-evaluation @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-evaluation.git@v0.1.8
|
||||
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.8
|
||||
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.8
|
||||
|
||||
99
tests/test_release_catalog_publication.py
Normal file
99
tests/test_release_catalog_publication.py
Normal file
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[1]
|
||||
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||
|
||||
from govoplan_release.publisher import publish_catalog_candidate # noqa: E402
|
||||
|
||||
|
||||
class ReleaseCatalogPublicationTests(unittest.TestCase):
|
||||
def test_publication_requires_an_existing_trust_anchor(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
candidate = self._candidate(root, key="candidate-key")
|
||||
web_root = root / "website"
|
||||
web_root.mkdir()
|
||||
|
||||
with patch(
|
||||
"govoplan_release.publisher.validate_module_package_catalog",
|
||||
return_value={"valid": True, "warnings": [], "error": None},
|
||||
):
|
||||
result = publish_catalog_candidate(
|
||||
candidate_dir=candidate,
|
||||
web_root=web_root,
|
||||
workspace_root=root,
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", result.status)
|
||||
self.assertIn("publication trust anchor is missing", " ".join(result.notes))
|
||||
|
||||
def test_publication_rejects_rebinding_an_existing_key_id(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
candidate = self._candidate(root, key="replacement-key")
|
||||
web_root = root / "website"
|
||||
target_keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
|
||||
target_keyring.parent.mkdir(parents=True)
|
||||
target_keyring.write_text(json.dumps(self._keyring("trusted-key")))
|
||||
|
||||
with patch(
|
||||
"govoplan_release.publisher.validate_module_package_catalog",
|
||||
return_value={"valid": True, "warnings": [], "error": None},
|
||||
):
|
||||
result = publish_catalog_candidate(
|
||||
candidate_dir=candidate,
|
||||
web_root=web_root,
|
||||
workspace_root=root,
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", result.status)
|
||||
self.assertIn("changes the public key", " ".join(result.notes))
|
||||
|
||||
@staticmethod
|
||||
def _candidate(root: Path, *, key: str) -> Path:
|
||||
candidate = root / "candidate"
|
||||
channel = candidate / "channels"
|
||||
channel.mkdir(parents=True)
|
||||
channel.joinpath("stable.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"channel": "stable",
|
||||
"core_release": {
|
||||
"version": "1.2.3",
|
||||
"python_ref": (
|
||||
"govoplan-core @ git+ssh://git@example.test/acme/"
|
||||
"govoplan-core.git@v1.2.3"
|
||||
),
|
||||
},
|
||||
"modules": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
candidate.joinpath("keyring.json").write_text(json.dumps(ReleaseCatalogPublicationTests._keyring(key)))
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def _keyring(key: str) -> dict[str, object]:
|
||||
return {
|
||||
"keys": [
|
||||
{
|
||||
"key_id": "release-key",
|
||||
"status": "active",
|
||||
"public_key": key,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -51,6 +51,22 @@ class ReleaseDoctorTests(unittest.TestCase):
|
||||
self.assertEqual("repo-dirty", findings[0].check_id)
|
||||
self.assertIn("git status --short", [item.command for item in findings[0].commands])
|
||||
|
||||
def test_repo_version_mismatch_blocks_release(self) -> None:
|
||||
doctor = load_doctor_module()
|
||||
repo = doctor.RepoState(
|
||||
name="govoplan-files",
|
||||
path="/tmp/govoplan-files",
|
||||
exists=True,
|
||||
pyproject_version="0.1.9",
|
||||
package_version="0.1.8",
|
||||
webui_package_version="0.1.9",
|
||||
)
|
||||
|
||||
findings = doctor.repo_findings((repo,), target_version="0.1.9", target_tag="v0.1.9")
|
||||
|
||||
mismatch = next(item for item in findings if item.check_id == "repo-version-mismatch")
|
||||
self.assertEqual("blocker", mismatch.severity)
|
||||
|
||||
def test_dubious_ownership_errors_suggest_safe_directory_command(self) -> None:
|
||||
doctor = load_doctor_module()
|
||||
repo = doctor.RepoState(
|
||||
|
||||
67
tests/test_release_entrypoint_gates.py
Normal file
67
tests/test_release_entrypoint_gates.py
Normal file
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class ReleaseEntrypointGateTests(unittest.TestCase):
|
||||
def test_lockstep_release_includes_every_registered_module_and_connector(self) -> None:
|
||||
script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text()
|
||||
registry = json.loads((META_ROOT / "repositories.json").read_text())
|
||||
|
||||
expected = {
|
||||
item["name"]
|
||||
for item in registry["repositories"]
|
||||
if item["category"] not in {"system", "website"}
|
||||
}
|
||||
missing = sorted(repo for repo in expected if f'$PARENT/{repo}"' not in script)
|
||||
|
||||
self.assertEqual([], missing)
|
||||
|
||||
def test_lockstep_release_runs_source_and_full_gates_before_remote_push(self) -> None:
|
||||
script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text()
|
||||
workflow = script.rsplit("\nconfirm_release\n", 1)[1]
|
||||
|
||||
source_gate = workflow.index("run_version_alignment_gate source")
|
||||
first_commit = workflow.index('run git -C "$repo" commit')
|
||||
lock_generation = workflow.index("generate_release_lock")
|
||||
full_gate = workflow.index("run_version_alignment_gate", source_gate + 1)
|
||||
first_push = workflow.index('run git -C "$repo" push')
|
||||
|
||||
self.assertLess(source_gate, first_commit)
|
||||
self.assertLess(first_commit, lock_generation)
|
||||
self.assertLess(lock_generation, full_gate)
|
||||
self.assertLess(full_gate, first_push)
|
||||
|
||||
def test_source_catalog_generator_enforces_explicit_repo_versions(self) -> None:
|
||||
script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text()
|
||||
|
||||
gate = script.index("selected_repository_version_issues(")
|
||||
write = script.index("output.write_text(")
|
||||
|
||||
self.assertLess(gate, write)
|
||||
|
||||
def test_candidate_publication_uses_existing_keyring_as_trust_anchor(self) -> None:
|
||||
publisher = (META_ROOT / "tools" / "release" / "govoplan_release" / "publisher.py").read_text()
|
||||
|
||||
self.assertIn("candidate_catalog_version_issues(candidate_payload)", publisher)
|
||||
self.assertIn("trusted_keys_from_keyring(\n target_keyring_payload", publisher)
|
||||
self.assertIn("trusted_keys=publication_trust", publisher)
|
||||
self.assertNotIn("trusted_keys=candidate_keys", publisher)
|
||||
self.assertIn("write_module_directory(", publisher)
|
||||
self.assertNotIn("shutil.copytree(candidate_modules", publisher)
|
||||
|
||||
def test_release_integration_honors_independent_module_versions(self) -> None:
|
||||
script = (META_ROOT / "tools" / "checks" / "check-release-integration.sh").read_text()
|
||||
|
||||
self.assertNotIn('expected_tag = os.environ["RELEASE_TAG"]', script)
|
||||
self.assertIn('dependency_version = tag_match.group(1)', script)
|
||||
self.assertIn('repo_tag="$(release_tag_for_package "$repo")"', script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
96
tests/test_release_integration.py
Normal file
96
tests/test_release_integration.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from tools.checks.release_integration import (
|
||||
SOURCE_COUPLED_CORE_TESTS,
|
||||
InstalledModuleArtifact,
|
||||
artifact_contract_issues,
|
||||
filter_test_suite,
|
||||
release_package_names,
|
||||
)
|
||||
|
||||
|
||||
def _named_test(test_id: str) -> unittest.TestCase:
|
||||
class NamedTest(unittest.TestCase):
|
||||
def id(self) -> str:
|
||||
return test_id
|
||||
|
||||
def runTest(self) -> None:
|
||||
return None
|
||||
|
||||
return NamedTest()
|
||||
|
||||
|
||||
class ReleaseIntegrationTests(unittest.TestCase):
|
||||
def test_release_requirement_parser_selects_only_immutable_module_refs(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
requirements = Path(temp_dir) / "requirements-release.txt"
|
||||
requirements.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
"../govoplan-core[server]",
|
||||
"govoplan-idm @ git+ssh://git@example.test/add-ideas/govoplan-idm.git@v0.1.8",
|
||||
"govoplan-campaign @ git+ssh://git@example.test/add-ideas/govoplan-campaign.git@v0.1.8",
|
||||
"govoplan-idm @ git+ssh://git@example.test/add-ideas/govoplan-idm.git@v0.1.8",
|
||||
"other-package==1.0",
|
||||
)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
release_package_names(requirements),
|
||||
("govoplan-idm", "govoplan-campaign"),
|
||||
)
|
||||
|
||||
def test_artifact_contracts_compare_distribution_entry_point_and_discovery_versions(self) -> None:
|
||||
artifacts = (
|
||||
InstalledModuleArtifact("govoplan-access", "0.1.8", "access", "0.1.8"),
|
||||
InstalledModuleArtifact("govoplan-idm", "0.1.8", "idm", "0.1.7"),
|
||||
InstalledModuleArtifact("govoplan-shadow-idm", "0.1.8", "idm", "0.1.8"),
|
||||
)
|
||||
|
||||
issues = artifact_contract_issues(
|
||||
artifacts,
|
||||
release_packages=("govoplan-access", "govoplan-idm", "govoplan-shadow-idm", "govoplan-campaign"),
|
||||
available_manifest_versions={"access": "0.1.8", "idm": "0.1.6"},
|
||||
)
|
||||
|
||||
self.assertEqual(len(issues), 5)
|
||||
self.assertTrue(any("metadata version '0.1.8' does not match" in issue for issue in issues))
|
||||
self.assertTrue(any("does not publish a govoplan.modules entry point" in issue for issue in issues))
|
||||
self.assertTrue(any("multiple release packages" in issue for issue in issues))
|
||||
|
||||
def test_release_suite_excludes_only_named_source_coupled_tests(self) -> None:
|
||||
retained_id = "tests.test_core_events.CoreEventTests.test_event_delivery"
|
||||
nested = unittest.TestSuite(
|
||||
(
|
||||
unittest.TestSuite(_named_test(test_id) for test_id in SOURCE_COUPLED_CORE_TESTS),
|
||||
unittest.TestSuite((_named_test(retained_id),)),
|
||||
)
|
||||
)
|
||||
|
||||
filtered, removed = filter_test_suite(nested, tuple(SOURCE_COUPLED_CORE_TESTS))
|
||||
|
||||
self.assertEqual(set(removed), set(SOURCE_COUPLED_CORE_TESTS))
|
||||
self.assertEqual([test.id() for test in filtered], [retained_id])
|
||||
self.assertEqual(len(SOURCE_COUPLED_CORE_TESTS), 4)
|
||||
|
||||
def test_release_entrypoint_uses_artifact_checks_and_filtered_core_suite(self) -> None:
|
||||
meta_root = Path(__file__).resolve().parents[1]
|
||||
script = (meta_root / "tools" / "checks" / "check-release-integration.sh").read_text(encoding="utf-8")
|
||||
|
||||
artifact_check = script.index('"$META_ROOT/tools/checks/release_integration.py" artifacts')
|
||||
core_tests = script.index('"$META_ROOT/tools/checks/release_integration.py" core-tests')
|
||||
module_tests = script.index('run_step "Run cloned module backend tests"')
|
||||
|
||||
self.assertLess(artifact_check, core_tests)
|
||||
self.assertLess(core_tests, module_tests)
|
||||
self.assertNotIn('"$PYTHON" -m unittest \\\n tests.test_module_system', script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
126
tests/test_release_sbom.py
Normal file
126
tests/test_release_sbom.py
Normal file
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[1]
|
||||
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||
|
||||
from govoplan_release.sbom import build_release_sbom, render_sbom, sbom_sha256, validate_sbom # noqa: E402
|
||||
|
||||
|
||||
class ReleaseSbomTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _build_at(timestamp: datetime) -> dict:
|
||||
return build_release_sbom(
|
||||
product_version="1.2.3",
|
||||
pip_inspect={
|
||||
"version": "1",
|
||||
"installed": [{"metadata": {"name": "FastAPI", "version": "0.100.0"}}],
|
||||
},
|
||||
npm_lock={"packages": {"": {"name": "govoplan", "version": "1.2.3"}}},
|
||||
timestamp=timestamp,
|
||||
)
|
||||
|
||||
def test_builds_valid_deduplicated_python_and_npm_components(self) -> None:
|
||||
integrity = base64.b64encode(b"example digest").decode("ascii")
|
||||
payload = build_release_sbom(
|
||||
product_version="1.2.3",
|
||||
pip_inspect={
|
||||
"version": "1",
|
||||
"installed": [
|
||||
{"metadata": {"name": "FastAPI", "version": "0.100.0"}},
|
||||
{"metadata": {"name": "FastAPI", "version": "0.100.0"}},
|
||||
],
|
||||
},
|
||||
npm_lock={
|
||||
"packages": {
|
||||
"": {"name": "govoplan", "version": "1.2.3"},
|
||||
"node_modules/@scope/example": {
|
||||
"name": "@scope/example",
|
||||
"version": "4.5.6",
|
||||
"integrity": f"sha512-{integrity}",
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
validate_sbom(payload)
|
||||
|
||||
self.assertEqual(2, len(payload["components"]))
|
||||
npm = next(item for item in payload["components"] if item["name"] == "@scope/example")
|
||||
self.assertEqual("SHA-512", npm["hashes"][0]["alg"])
|
||||
self.assertIn("pkg:npm/%40scope/example@4.5.6", npm["purl"])
|
||||
self.assertEqual(
|
||||
hashlib.sha256(render_sbom(payload).encode("utf-8")).hexdigest(),
|
||||
sbom_sha256(payload),
|
||||
)
|
||||
|
||||
def test_rejects_empty_component_inventory(self) -> None:
|
||||
payload = build_release_sbom(
|
||||
product_version="1.2.3",
|
||||
pip_inspect={"version": "1", "installed": []},
|
||||
npm_lock={"packages": {"": {"version": "1.2.3"}}},
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "no dependency components"):
|
||||
validate_sbom(payload)
|
||||
|
||||
def test_fixed_timestamp_produces_identical_bytes_and_digest(self) -> None:
|
||||
timestamp = datetime(2026, 7, 21, 8, 30, tzinfo=UTC)
|
||||
|
||||
first = self._build_at(timestamp)
|
||||
second = self._build_at(timestamp)
|
||||
|
||||
self.assertEqual(render_sbom(first), render_sbom(second))
|
||||
self.assertEqual(sbom_sha256(first), sbom_sha256(second))
|
||||
|
||||
def test_timestamp_changes_serial_and_digest(self) -> None:
|
||||
first = self._build_at(datetime(2026, 7, 21, 8, 30, tzinfo=UTC))
|
||||
second = self._build_at(datetime(2026, 7, 21, 8, 31, tzinfo=UTC))
|
||||
|
||||
self.assertNotEqual(first["serialNumber"], second["serialNumber"])
|
||||
self.assertNotEqual(sbom_sha256(first), sbom_sha256(second))
|
||||
|
||||
def test_local_and_credential_bearing_distribution_urls_are_omitted(self) -> None:
|
||||
payload = build_release_sbom(
|
||||
product_version="1.2.3",
|
||||
pip_inspect={
|
||||
"version": "1",
|
||||
"installed": [
|
||||
{
|
||||
"metadata": {"name": "LocalPackage", "version": "1.0.0"},
|
||||
"direct_url": {"url": "file:///private/build/govoplan-core"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "PrivatePackage", "version": "1.0.0"},
|
||||
"direct_url": {"url": "https://user:secret@example.test/private.whl"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "PublicPackage", "version": "1.0.0"},
|
||||
"direct_url": {"url": "https://example.test/public.whl"},
|
||||
},
|
||||
],
|
||||
},
|
||||
npm_lock={"packages": {"": {"name": "govoplan", "version": "1.2.3"}}},
|
||||
timestamp=datetime(2026, 7, 21, 8, 30, tzinfo=UTC),
|
||||
)
|
||||
|
||||
by_name = {component["name"]: component for component in payload["components"]}
|
||||
self.assertNotIn("externalReferences", by_name["LocalPackage"])
|
||||
self.assertNotIn("externalReferences", by_name["PrivatePackage"])
|
||||
self.assertEqual(
|
||||
[{"type": "distribution", "url": "https://example.test/public.whl"}],
|
||||
by_name["PublicPackage"]["externalReferences"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
358
tests/test_security_audit_wrapper.py
Normal file
358
tests/test_security_audit_wrapper.py
Normal file
@@ -0,0 +1,358 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[1]
|
||||
AUDIT_SCRIPT = META_ROOT / "tools" / "checks" / "check-security-audit.sh"
|
||||
|
||||
|
||||
class SecurityAuditWrapperTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._temporary_directory = tempfile.TemporaryDirectory()
|
||||
self.temporary_root = Path(self._temporary_directory.name)
|
||||
self.checkout = self.temporary_root / "checkout"
|
||||
checks = self.checkout / "tools" / "checks"
|
||||
checks.mkdir(parents=True)
|
||||
shutil.copy2(AUDIT_SCRIPT, checks / AUDIT_SCRIPT.name)
|
||||
(self.checkout / "README.md").write_text("audit fixture\n", encoding="utf-8")
|
||||
|
||||
subprocess.run(["git", "init", "-q", str(self.checkout)], check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(self.checkout),
|
||||
"config",
|
||||
"user.email",
|
||||
"audit@example.invalid",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "-C", str(self.checkout), "config", "user.name", "Audit Test"],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(["git", "-C", str(self.checkout), "add", "."], check=True)
|
||||
subprocess.run(
|
||||
["git", "-C", str(self.checkout), "commit", "-qm", "audit fixture"],
|
||||
check=True,
|
||||
)
|
||||
|
||||
self.stub_bin = self.temporary_root / "bin"
|
||||
self.stub_bin.mkdir()
|
||||
self._write_stub(
|
||||
"semgrep",
|
||||
r"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo 'semgrep stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
if [[ " $* " != *" --error "* ]]; then
|
||||
echo 'missing --error' >&2
|
||||
exit 3
|
||||
fi
|
||||
output=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == '--output' ]]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
else
|
||||
shift
|
||||
fi
|
||||
done
|
||||
case "${STUB_SEMGREP_REPORT:-valid}" in
|
||||
valid)
|
||||
printf '%s\n' '{"version":"2.1.0","runs":[]}' > "$output"
|
||||
;;
|
||||
malformed)
|
||||
printf '%s\n' '{' > "$output"
|
||||
;;
|
||||
invalid-sarif)
|
||||
printf '%s\n' '{}' > "$output"
|
||||
;;
|
||||
missing)
|
||||
;;
|
||||
esac
|
||||
if [[ -n "${STUB_MUTATE_PATH:-}" ]]; then
|
||||
printf '%s\n' 'changed during audit' >> "$STUB_MUTATE_PATH"
|
||||
fi
|
||||
exit "${STUB_SEMGREP_EXIT:-0}"
|
||||
""",
|
||||
)
|
||||
self._write_stub(
|
||||
"gitleaks",
|
||||
r"""
|
||||
if [[ "${1:-}" == '--version' || "${1:-}" == 'version' ]]; then
|
||||
echo 'gitleaks stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${1:-}" == 'git' && "${2:-}" == '--help' ]]; then
|
||||
exit 0
|
||||
fi
|
||||
output=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == '--report-path' ]]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
else
|
||||
shift
|
||||
fi
|
||||
done
|
||||
mkdir -p "$(dirname "$output")"
|
||||
printf '%s\n' '[]' > "$output"
|
||||
exit "${STUB_GITLEAKS_EXIT:-0}"
|
||||
""",
|
||||
)
|
||||
for tool in ("bandit", "ruff"):
|
||||
self._write_stub(
|
||||
tool,
|
||||
f"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo '{tool} stub 1.0'
|
||||
fi
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._temporary_directory.cleanup()
|
||||
|
||||
def _write_stub(self, name: str, body: str) -> None:
|
||||
path = self.stub_bin / name
|
||||
path.write_text(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\n" + textwrap.dedent(body).lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path.chmod(0o755)
|
||||
|
||||
def _install_full_mode_stubs(self) -> None:
|
||||
self._write_stub(
|
||||
"trivy",
|
||||
r"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo 'trivy stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
output=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == '--output' ]]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
else
|
||||
shift
|
||||
fi
|
||||
done
|
||||
printf '%s\n' '{"version":"2.1.0","runs":[]}' > "$output"
|
||||
""",
|
||||
)
|
||||
self._write_stub(
|
||||
"osv-scanner",
|
||||
r"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo 'osv-scanner stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
if [[ " $* " == *" scan --help "* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
output=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == '--output-file' ]]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
else
|
||||
shift
|
||||
fi
|
||||
done
|
||||
printf '{"results": %s}\n' "${STUB_OSV_RESULTS:-null}" > "$output"
|
||||
""",
|
||||
)
|
||||
self._write_stub(
|
||||
"jscpd",
|
||||
r"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo 'jscpd stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
output=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == '--output' ]]; then
|
||||
output="$2"
|
||||
shift 2
|
||||
else
|
||||
shift
|
||||
fi
|
||||
done
|
||||
mkdir -p "$output"
|
||||
printf '%s\n' '{"duplicates":[],"statistics":{}}' > "$output/jscpd-report.json"
|
||||
""",
|
||||
)
|
||||
for tool in ("pip-audit", "npm", "radon", "xenon"):
|
||||
self._write_stub(
|
||||
tool,
|
||||
f"""
|
||||
if [[ " $* " == *" --version "* ]]; then
|
||||
echo '{tool} stub 1.0'
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
|
||||
def _run(
|
||||
self, *arguments: str, mode: str = "quick", **environment: str
|
||||
) -> tuple[subprocess.CompletedProcess[str], Path]:
|
||||
reports = self.temporary_root / "reports"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"PATH": f"{self.stub_bin}:{env['PATH']}",
|
||||
"SECURITY_AUDIT_REQUIRE_TOOLS": "1",
|
||||
"SECURITY_AUDIT_TOOLBOX_FINGERPRINT": 'stub-"safe',
|
||||
**environment,
|
||||
}
|
||||
)
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(self.checkout / "tools" / "checks" / AUDIT_SCRIPT.name),
|
||||
"--mode",
|
||||
mode,
|
||||
"--scope",
|
||||
"current",
|
||||
"--reports-dir",
|
||||
str(reports),
|
||||
*arguments,
|
||||
],
|
||||
cwd=self.checkout,
|
||||
env=env,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result, reports
|
||||
|
||||
@staticmethod
|
||||
def _manifest(reports: Path) -> dict[str, object]:
|
||||
return json.loads((reports / "manifest.json").read_text(encoding="utf-8"))
|
||||
|
||||
def test_report_only_findings_are_nonfatal_and_strict_findings_fail(self) -> None:
|
||||
result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="1")
|
||||
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
manifest = self._manifest(reports)
|
||||
self.assertEqual(0, manifest["overall_status"])
|
||||
self.assertEqual(1, manifest["finding_status"])
|
||||
self.assertEqual(0, manifest["execution_error_status"])
|
||||
self.assertIn(
|
||||
"Semgrep SAST\t1\tfindings", (reports / "step-status.tsv").read_text()
|
||||
)
|
||||
|
||||
strict_result, strict_reports = self._run("--strict", STUB_SEMGREP_EXIT="1")
|
||||
self.assertEqual(1, strict_result.returncode)
|
||||
self.assertEqual(1, self._manifest(strict_reports)["overall_status"])
|
||||
|
||||
def test_scanner_execution_error_is_fatal_in_report_only_mode(self) -> None:
|
||||
result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="2")
|
||||
|
||||
self.assertEqual(1, result.returncode)
|
||||
manifest = self._manifest(reports)
|
||||
self.assertEqual(1, manifest["overall_status"])
|
||||
self.assertEqual(1, manifest["execution_error_status"])
|
||||
self.assertIn(
|
||||
"Semgrep SAST\t2\texecution-error",
|
||||
(reports / "step-status.tsv").read_text(),
|
||||
)
|
||||
|
||||
def test_malformed_or_missing_machine_report_is_fatal(self) -> None:
|
||||
for report_kind in ("malformed", "invalid-sarif", "missing"):
|
||||
with self.subTest(report_kind=report_kind):
|
||||
result, reports = self._run(
|
||||
"--report-only",
|
||||
STUB_SEMGREP_REPORT=report_kind,
|
||||
)
|
||||
self.assertEqual(1, result.returncode)
|
||||
manifest = self._manifest(reports)
|
||||
self.assertEqual(1, manifest["execution_error_status"])
|
||||
if report_kind == "missing":
|
||||
self.assertIn("semgrep.sarif", manifest["missing_reports"])
|
||||
self.assertIn(
|
||||
"Validate machine-readable audit reports\t2\texecution-error",
|
||||
(reports / "step-status.tsv").read_text(),
|
||||
)
|
||||
|
||||
def test_osv_null_results_are_valid_but_wrong_shapes_fail(self) -> None:
|
||||
self._install_full_mode_stubs()
|
||||
|
||||
result, reports = self._run("--report-only", mode="full")
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
osv_report = json.loads(
|
||||
(reports / "osv-scanner-checkout.json").read_text(encoding="utf-8")
|
||||
)
|
||||
self.assertIsNone(osv_report["results"])
|
||||
|
||||
invalid_result, invalid_reports = self._run(
|
||||
"--report-only",
|
||||
mode="full",
|
||||
STUB_OSV_RESULTS="{}",
|
||||
)
|
||||
self.assertEqual(1, invalid_result.returncode)
|
||||
self.assertIn(
|
||||
"invalid OSV-Scanner report",
|
||||
invalid_result.stderr,
|
||||
)
|
||||
self.assertEqual(1, self._manifest(invalid_reports)["execution_error_status"])
|
||||
|
||||
def test_workspace_mutation_invalidates_the_audit(self) -> None:
|
||||
result, reports = self._run(
|
||||
"--report-only",
|
||||
STUB_MUTATE_PATH=str(self.checkout / "README.md"),
|
||||
)
|
||||
|
||||
self.assertEqual(1, result.returncode)
|
||||
manifest = self._manifest(reports)
|
||||
self.assertFalse(manifest["workspace_unchanged"])
|
||||
self.assertEqual(1, manifest["overall_status"])
|
||||
self.assertNotEqual(
|
||||
(reports / "workspace-state-start.tsv").read_text(),
|
||||
(reports / "workspace-state-end.tsv").read_text(),
|
||||
)
|
||||
|
||||
def test_manifest_and_checksums_cover_only_current_run_reports(self) -> None:
|
||||
reports = self.temporary_root / "reports"
|
||||
reports.mkdir()
|
||||
(reports / "stale-malformed.json").write_text("{", encoding="utf-8")
|
||||
|
||||
result, reports = self._run("--report-only")
|
||||
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
manifest = self._manifest(reports)
|
||||
self.assertEqual('stub-"safe', manifest["toolbox_fingerprint"])
|
||||
self.assertNotIn("stale-malformed.json", manifest["reports"])
|
||||
self.assertEqual([], manifest["missing_reports"])
|
||||
checksums = (reports / "report-sha256.txt").read_text(encoding="utf-8")
|
||||
self.assertNotIn("stale-malformed.json", checksums)
|
||||
verification = subprocess.run(
|
||||
["sha256sum", "--check", "report-sha256.txt"],
|
||||
cwd=reports,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(0, verification.returncode, verification.stderr)
|
||||
checksummed_paths = {
|
||||
line.split(" ", 1)[1] for line in checksums.splitlines() if " " in line
|
||||
}
|
||||
self.assertEqual(set(manifest["reports"]), checksummed_paths)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
225
tests/test_version_alignment.py
Normal file
225
tests/test_version_alignment.py
Normal file
@@ -0,0 +1,225 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[1]
|
||||
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||
|
||||
from govoplan_release.version_alignment import ( # noqa: E402
|
||||
candidate_catalog_version_issues,
|
||||
release_composition_issues,
|
||||
repository_version_issues,
|
||||
selected_repository_version_issues,
|
||||
)
|
||||
from govoplan_release.model import RepositorySnapshot, RepositorySpec, VersionSnapshot # noqa: E402
|
||||
from govoplan_release.selective_planner import build_unit # noqa: E402
|
||||
from govoplan_release.selective_catalog import enforce_selected_version_alignment # noqa: E402
|
||||
|
||||
|
||||
class VersionAlignmentTests(unittest.TestCase):
|
||||
def test_candidate_catalog_refs_and_selected_units_must_match_versions(self) -> None:
|
||||
payload = {
|
||||
"core_release": {
|
||||
"version": "1.2.3",
|
||||
"python_ref": "govoplan-core @ git+ssh://git@example.test/acme/govoplan-core.git@v1.2.3",
|
||||
},
|
||||
"modules": [
|
||||
{
|
||||
"module_id": "files",
|
||||
"version": "1.2.3",
|
||||
"python_ref": "govoplan-files @ git+ssh://git@example.test/acme/govoplan-files.git@v1.2.4",
|
||||
"webui_package": "@govoplan/files-webui",
|
||||
"webui_ref": "git+ssh://git@example.test/acme/govoplan-files.git#v1.2.3",
|
||||
}
|
||||
],
|
||||
"release": {
|
||||
"selected_units": [
|
||||
{"repo": "govoplan-files", "version": "1.2.5", "tag": "v1.2.6"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
issues = candidate_catalog_version_issues(payload)
|
||||
|
||||
messages = {issue.message for issue in issues}
|
||||
self.assertIn("Python ref tag must match the catalog entry version", messages)
|
||||
self.assertIn("selected-unit tag must match its version", messages)
|
||||
self.assertIn("selected-unit version must match the catalog entry", messages)
|
||||
|
||||
def test_catalog_candidate_gate_rejects_selected_repo_drift(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workspace = Path(tmp)
|
||||
repo = workspace / "govoplan-files"
|
||||
(repo / "webui").mkdir(parents=True)
|
||||
(repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n')
|
||||
(repo / "webui" / "package.json").write_text('{"version":"1.2.4"}\n')
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "Version alignment gate failed"):
|
||||
enforce_selected_version_alignment(
|
||||
repo_versions={"govoplan-files": "1.2.5"},
|
||||
workspace=workspace,
|
||||
)
|
||||
|
||||
def test_catalog_candidate_gate_rejects_requested_version_that_differs_from_source(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workspace = Path(tmp)
|
||||
repo = workspace / "govoplan-files"
|
||||
(repo / "webui").mkdir(parents=True)
|
||||
(repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n')
|
||||
(repo / "webui" / "package.json").write_text('{"version":"1.2.3"}\n')
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "source version must match the requested release version"):
|
||||
enforce_selected_version_alignment(
|
||||
repo_versions={"govoplan-files": "1.2.4"},
|
||||
workspace=workspace,
|
||||
)
|
||||
|
||||
def test_selected_repository_gate_reports_missing_version_metadata(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workspace = Path(tmp)
|
||||
(workspace / "govoplan-files").mkdir()
|
||||
|
||||
issues = selected_repository_version_issues(
|
||||
repo_versions={"govoplan-files": "1.2.3"},
|
||||
workspace=workspace,
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(issues))
|
||||
self.assertEqual("repository has no version metadata", issues[0].message)
|
||||
|
||||
def test_selective_release_unit_is_blocked_by_version_drift(self) -> None:
|
||||
repo = RepositorySnapshot(
|
||||
spec=RepositorySpec(
|
||||
name="govoplan-example",
|
||||
category="module",
|
||||
subtype="domain",
|
||||
remote="git@example.test:acme/govoplan-example.git",
|
||||
path="govoplan-example",
|
||||
),
|
||||
absolute_path="/tmp/govoplan-example",
|
||||
exists=True,
|
||||
is_git=True,
|
||||
has_head=True,
|
||||
versions=VersionSnapshot(pyproject="1.2.3", webui_package="1.2.4", manifests=("1.2.3",)),
|
||||
)
|
||||
|
||||
unit = build_unit(repo, target_version="1.2.5", contracts=None)
|
||||
|
||||
self.assertEqual("blocked", unit.status)
|
||||
self.assertIn("version metadata is not aligned", unit.blockers[0])
|
||||
|
||||
def test_repository_versions_and_lockfiles_must_match(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp) / "govoplan-example"
|
||||
(root / "src" / "govoplan_example" / "backend").mkdir(parents=True)
|
||||
(root / "webui").mkdir()
|
||||
(root / "pyproject.toml").write_text('[project]\nname = "govoplan-example"\nversion = "1.2.3"\n')
|
||||
(root / "package.json").write_text('{"name":"@govoplan/example","version":"1.2.3"}\n')
|
||||
(root / "webui" / "package.json").write_text('{"name":"@govoplan/example-webui","version":"1.2.4"}\n')
|
||||
(root / "webui" / "package-lock.json").write_text(
|
||||
json.dumps({"packages": {"": {"version": "1.2.2"}}})
|
||||
)
|
||||
(root / "src" / "govoplan_example" / "backend" / "manifest.py").write_text(
|
||||
'manifest = ModuleManifest(\n version="1.2.3",\n)\n'
|
||||
)
|
||||
|
||||
issues = repository_version_issues(root)
|
||||
|
||||
self.assertEqual(2, len(issues))
|
||||
self.assertEqual({"webui/package.json", "webui/package-lock.json"}, {issue.source for issue in issues})
|
||||
|
||||
def test_public_runtime_version_must_match_package_metadata(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp) / "govoplan-example"
|
||||
package = root / "src" / "govoplan_example"
|
||||
package.mkdir(parents=True)
|
||||
(root / "pyproject.toml").write_text(
|
||||
'[project]\nname = "govoplan-example"\nversion = "1.2.3"\n'
|
||||
)
|
||||
(package / "__init__.py").write_text('__version__ = "1.2.2"\n')
|
||||
|
||||
issues = repository_version_issues(root)
|
||||
|
||||
self.assertEqual(1, len(issues))
|
||||
self.assertEqual("package __init__.py 1", issues[0].source)
|
||||
self.assertEqual("1.2.3", issues[0].expected)
|
||||
self.assertEqual("1.2.2", issues[0].actual)
|
||||
|
||||
def test_release_backend_and_frontend_refs_must_match(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
meta = Path(tmp) / "govoplan"
|
||||
core = Path(tmp) / "govoplan-core"
|
||||
meta.mkdir()
|
||||
(core / "webui").mkdir(parents=True)
|
||||
(meta / "requirements-release.txt").write_text(
|
||||
"govoplan-example @ git+ssh://git@example.test/acme/govoplan-example.git@v1.2.3\n"
|
||||
)
|
||||
(core / "webui" / "package.release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dependencies": {
|
||||
"@govoplan/example-webui": (
|
||||
"git+ssh://git@example.test/acme/govoplan-example.git#v1.2.4"
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
issues = release_composition_issues(meta, core_root=core)
|
||||
|
||||
self.assertEqual(1, len(issues))
|
||||
self.assertEqual("1.2.3", issues[0].expected)
|
||||
self.assertEqual("1.2.4", issues[0].actual)
|
||||
|
||||
def test_release_refs_must_point_at_installable_versioned_artifacts(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workspace = Path(tmp)
|
||||
meta = workspace / "govoplan"
|
||||
core = workspace / "govoplan-core"
|
||||
module = workspace / "govoplan-example"
|
||||
meta.mkdir()
|
||||
(core / "webui").mkdir(parents=True)
|
||||
module.mkdir()
|
||||
(meta / "requirements-release.txt").write_text(
|
||||
"govoplan-example @ git+ssh://git@example.test/acme/govoplan-example.git@v1.2.3\n"
|
||||
)
|
||||
(core / "webui" / "package.release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dependencies": {
|
||||
"@govoplan/example-webui": (
|
||||
"git+ssh://git@example.test/acme/govoplan-example.git#v1.2.3"
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
subprocess.run(["git", "init", "-q", str(module)], check=True)
|
||||
subprocess.run(["git", "-C", str(module), "config", "user.email", "test@example.test"], check=True)
|
||||
subprocess.run(["git", "-C", str(module), "config", "user.name", "Test"], check=True)
|
||||
(module / "README.md").write_text("uninstallable release\n")
|
||||
subprocess.run(["git", "-C", str(module), "add", "README.md"], check=True)
|
||||
subprocess.run(["git", "-C", str(module), "commit", "-qm", "release"], check=True)
|
||||
subprocess.run(["git", "-C", str(module), "tag", "v1.2.3"], check=True)
|
||||
|
||||
issues = release_composition_issues(meta, core_root=core)
|
||||
|
||||
self.assertEqual(2, len(issues))
|
||||
self.assertEqual(
|
||||
{"v1.2.3:pyproject.toml", "v1.2.3:webui/package.json"},
|
||||
{issue.source for issue in issues},
|
||||
)
|
||||
self.assertTrue(all("must contain" in issue.message for issue in issues))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -32,9 +32,8 @@ with (Path.cwd() / "pyproject.toml").open("rb") as handle:
|
||||
print(tomllib.load(handle)["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
RELEASE_TAG="${GOVOPLAN_RELEASE_TAG:-v$RELEASE_VERSION}"
|
||||
|
||||
export RELEASE_TAG RELEASE_VERSION
|
||||
export RELEASE_VERSION
|
||||
export META_ROOT
|
||||
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
|
||||
export GOVOPLAN_MIGRATION_TRACK=release
|
||||
@@ -91,7 +90,32 @@ install_cloned_webui_dependencies() {
|
||||
)
|
||||
}
|
||||
|
||||
release_tag_for_package() {
|
||||
local package_name="$1"
|
||||
"$PYTHON" - "$META_ROOT/requirements-release.txt" "$package_name" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
requirements_path = Path(sys.argv[1])
|
||||
package_name = sys.argv[2]
|
||||
pattern = re.compile(
|
||||
rf"^{re.escape(package_name)}\s+@\s+git\+ssh://.+?\.git@(v[0-9]+\.[0-9]+\.[0-9]+)$"
|
||||
)
|
||||
for line in requirements_path.read_text(encoding="utf-8").splitlines():
|
||||
match = pattern.match(line.strip())
|
||||
if match:
|
||||
print(match.group(1))
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(f"No immutable release reference found for {package_name}")
|
||||
PY
|
||||
}
|
||||
|
||||
run_step "Validate release refs and installed package metadata"
|
||||
"$PYTHON" "$META_ROOT/tools/checks/check-version-alignment.py" --release-composition
|
||||
"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
@@ -108,7 +132,6 @@ root = Path.cwd()
|
||||
release_requirements = Path(os.environ["META_ROOT"]) / "requirements-release.txt"
|
||||
release_webui = root / "webui" / "package.release.json"
|
||||
expected_version = os.environ["RELEASE_VERSION"]
|
||||
expected_tag = os.environ["RELEASE_TAG"]
|
||||
errors: list[str] = []
|
||||
|
||||
python_requirements: list[tuple[str, str, str]] = []
|
||||
@@ -121,8 +144,6 @@ if not python_requirements:
|
||||
errors.append("No GovOPlaN git+ssh release requirements found.")
|
||||
|
||||
for package_name, repo_url, tag in python_requirements:
|
||||
if tag != expected_tag:
|
||||
errors.append(f"{package_name} release requirement uses {tag!r}, expected {expected_tag!r}.")
|
||||
version = metadata.version(package_name)
|
||||
if version != tag.removeprefix("v"):
|
||||
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
|
||||
@@ -146,15 +167,18 @@ for package_name, spec in sorted(webui_dependencies.items()):
|
||||
if not package_name.startswith("@govoplan/"):
|
||||
continue
|
||||
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
|
||||
if tag != expected_tag:
|
||||
errors.append(f"{package_name} release dependency uses {tag!r}, expected {expected_tag}.")
|
||||
tag_match = re.fullmatch(r"v([0-9]+\.[0-9]+\.[0-9]+)", tag)
|
||||
if tag_match is None:
|
||||
errors.append(f"{package_name} release dependency has invalid immutable tag {tag!r}.")
|
||||
continue
|
||||
dependency_version = tag_match.group(1)
|
||||
package_json = root / "webui" / "node_modules" / package_name / "package.json"
|
||||
if not package_json.exists():
|
||||
errors.append(f"{package_name} is missing from release node_modules.")
|
||||
continue
|
||||
installed = json.loads(package_json.read_text(encoding="utf-8"))
|
||||
if installed.get("version") != expected_version:
|
||||
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {expected_version!r}.")
|
||||
if installed.get("version") != dependency_version:
|
||||
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {dependency_version!r} from {tag!r}.")
|
||||
|
||||
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
|
||||
root_peers = release_package.get("peerDependencies", {})
|
||||
@@ -173,40 +197,14 @@ print(f"Release refs and metadata passed for {len(python_requirements)} Python p
|
||||
PY
|
||||
|
||||
run_step "Validate installed module manifests and registry"
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
"$PYTHON" "$META_ROOT/tools/checks/release_integration.py" artifacts \
|
||||
--requirements "$META_ROOT/requirements-release.txt"
|
||||
|
||||
import sys
|
||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||
|
||||
expected = {
|
||||
"access",
|
||||
"admin",
|
||||
"audit",
|
||||
"calendar",
|
||||
"campaigns",
|
||||
"dashboard",
|
||||
"docs",
|
||||
"files",
|
||||
"identity",
|
||||
"idm",
|
||||
"mail",
|
||||
"ops",
|
||||
"organizations",
|
||||
"policy",
|
||||
"tenancy",
|
||||
}
|
||||
|
||||
manifests = available_module_manifests()
|
||||
missing = sorted(expected - set(manifests))
|
||||
if missing:
|
||||
print(f"Missing installed module manifests: {', '.join(missing)}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
registry = build_platform_registry(tuple(sorted(expected)))
|
||||
snapshot = registry.validate()
|
||||
print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests))
|
||||
PY
|
||||
run_step "Generate release dependency provenance"
|
||||
"$PYTHON" "$META_ROOT/tools/release/generate-release-sbom.py" \
|
||||
--python "$PYTHON" \
|
||||
--core-root "$ROOT" \
|
||||
--output "$WORK_ROOT/govoplan-sbom.cdx.json"
|
||||
|
||||
run_step "Run SQLite migration smoke for release modules"
|
||||
"$PYTHON" - <<'PY'
|
||||
@@ -245,16 +243,13 @@ PY
|
||||
|
||||
run_step "Clone release module test sources"
|
||||
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
|
||||
git clone --depth 1 --branch "$RELEASE_TAG" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
||||
repo_tag="$(release_tag_for_package "$repo")"
|
||||
git clone --depth 1 --branch "$repo_tag" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
||||
done
|
||||
|
||||
run_step "Run core backend release tests"
|
||||
"$PYTHON" -m unittest \
|
||||
tests.test_module_system \
|
||||
tests.test_access_contracts \
|
||||
tests.test_identity_organization_contracts \
|
||||
tests.test_core_events \
|
||||
tests.test_policy_contracts
|
||||
"$PYTHON" "$META_ROOT/tools/checks/release_integration.py" core-tests \
|
||||
--core-root "$ROOT"
|
||||
"$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py"
|
||||
|
||||
run_step "Run cloned module backend tests"
|
||||
|
||||
@@ -77,6 +77,15 @@ if [[ "$REPORTS_DIR" != /* ]]; then
|
||||
REPORTS_DIR="$ROOT/$REPORTS_DIR"
|
||||
fi
|
||||
mkdir -p "$REPORTS_DIR"
|
||||
REPORTS_DIR="$(cd "$REPORTS_DIR" && pwd -P)"
|
||||
|
||||
for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do
|
||||
flag_value="${!flag_name}"
|
||||
if [[ "$flag_value" != "0" && "$flag_value" != "1" ]]; then
|
||||
echo "$flag_name must be 0 or 1, got: $flag_value" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
declare -a REPOS=()
|
||||
if [[ "$SCOPE" == "govoplan" ]]; then
|
||||
@@ -118,7 +127,7 @@ for repo in "${REPOS[@]}"; do
|
||||
done
|
||||
|
||||
safe_name() {
|
||||
basename "$1" | tr -c 'A-Za-z0-9_.-' '_'
|
||||
printf '%s' "$(basename "$1")" | tr -c 'A-Za-z0-9_.-' '_'
|
||||
}
|
||||
|
||||
has_tool() {
|
||||
@@ -128,58 +137,341 @@ has_tool() {
|
||||
overall_status=0
|
||||
finding_status=0
|
||||
missing_status=0
|
||||
execution_status=0
|
||||
audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
audit_completed_at=""
|
||||
workspace_unchanged="unknown"
|
||||
audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}"
|
||||
declare -A REPORT_FILES=()
|
||||
declare -A MACHINE_REPORTS=()
|
||||
|
||||
register_report() {
|
||||
local path="$1"
|
||||
local report_kind="${2:-text}"
|
||||
case "$path" in
|
||||
"$REPORTS_DIR"/*) ;;
|
||||
*)
|
||||
echo "Refusing to register an audit report outside $REPORTS_DIR: $path" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
REPORT_FILES["$path"]=1
|
||||
if [[ "$report_kind" == "machine" ]]; then
|
||||
MACHINE_REPORTS["$path"]=1
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_machine_report() {
|
||||
local path="$1"
|
||||
register_report "$path" machine || return 2
|
||||
if [[ -e "$path" || -L "$path" ]]; then
|
||||
rm -f -- "$path" || return 2
|
||||
fi
|
||||
}
|
||||
|
||||
run_step() {
|
||||
local name="$1"
|
||||
shift
|
||||
local exit_contract="$2"
|
||||
shift 2
|
||||
echo
|
||||
echo "==> $name"
|
||||
"$@"
|
||||
local status=$?
|
||||
if [[ "$status" -ne 0 ]]; then
|
||||
echo "Audit step reported findings or failed: $name (exit $status)" >&2
|
||||
local outcome="passed"
|
||||
if [[ "$status" -eq 0 ]]; then
|
||||
:
|
||||
elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then
|
||||
outcome="findings"
|
||||
echo "Audit step reported findings: $name (exit $status)" >&2
|
||||
finding_status=1
|
||||
if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
|
||||
overall_status=1
|
||||
fi
|
||||
else
|
||||
outcome="execution-error"
|
||||
echo "Audit step failed to execute successfully: $name (exit $status)" >&2
|
||||
execution_status=1
|
||||
overall_status=1
|
||||
fi
|
||||
printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv"
|
||||
}
|
||||
|
||||
skip_or_fail_missing() {
|
||||
local tool="$1"
|
||||
echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
|
||||
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
|
||||
missing_status=1
|
||||
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
|
||||
overall_status=1
|
||||
fi
|
||||
printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv"
|
||||
}
|
||||
|
||||
write_manifest() {
|
||||
{
|
||||
printf '{\n'
|
||||
printf ' "mode": "%s",\n' "$MODE"
|
||||
printf ' "scope": "%s",\n' "$SCOPE"
|
||||
printf ' "fail_on_findings": "%s",\n' "$FAIL_ON_FINDINGS"
|
||||
printf ' "repositories": [\n'
|
||||
local index=0
|
||||
local -a present_reports=()
|
||||
local -a expected_reports=()
|
||||
local path
|
||||
while IFS= read -r path; do
|
||||
expected_reports+=("${path#"$REPORTS_DIR"/}")
|
||||
if [[ -f "$path" ]]; then
|
||||
present_reports+=("${path#"$REPORTS_DIR"/}")
|
||||
fi
|
||||
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
||||
|
||||
python3 - \
|
||||
"$REPORTS_DIR/manifest.json" \
|
||||
"$MODE" \
|
||||
"$SCOPE" \
|
||||
"$FAIL_ON_FINDINGS" \
|
||||
"$audit_started_at" \
|
||||
"$audit_completed_at" \
|
||||
"$workspace_unchanged" \
|
||||
"$overall_status" \
|
||||
"$finding_status" \
|
||||
"$execution_status" \
|
||||
"$missing_status" \
|
||||
"$audit_toolbox_fingerprint" \
|
||||
"${#REPOS[@]}" \
|
||||
"${REPOS[@]}" \
|
||||
"${#present_reports[@]}" \
|
||||
"${present_reports[@]}" \
|
||||
"${expected_reports[@]}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
(
|
||||
manifest_path,
|
||||
mode,
|
||||
scope,
|
||||
fail_on_findings,
|
||||
started_at,
|
||||
completed_at,
|
||||
workspace_unchanged,
|
||||
overall_status,
|
||||
finding_status,
|
||||
execution_status,
|
||||
missing_status,
|
||||
toolbox_fingerprint,
|
||||
repository_count,
|
||||
*remaining,
|
||||
) = sys.argv[1:]
|
||||
repo_count = int(repository_count)
|
||||
repositories = remaining[:repo_count]
|
||||
remaining = remaining[repo_count:]
|
||||
report_count = int(remaining[0])
|
||||
reports = remaining[1 : report_count + 1]
|
||||
expected_reports = remaining[report_count + 1 :]
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"scope": scope,
|
||||
"fail_on_findings": fail_on_findings == "1",
|
||||
"started_at_utc": started_at,
|
||||
"completed_at_utc": completed_at,
|
||||
"toolbox_fingerprint": toolbox_fingerprint,
|
||||
"workspace_unchanged": workspace_unchanged == "true",
|
||||
"overall_status": int(overall_status),
|
||||
"finding_status": int(finding_status),
|
||||
"execution_error_status": int(execution_status),
|
||||
"missing_tool_status": int(missing_status),
|
||||
"tool_versions": "tool-versions.txt",
|
||||
"workspace_state_start": "workspace-state-start.tsv",
|
||||
"workspace_state_end": "workspace-state-end.tsv",
|
||||
"report_checksums": "report-sha256.txt",
|
||||
"step_status": "step-status.tsv",
|
||||
"repositories": repositories,
|
||||
"reports": reports,
|
||||
"expected_reports": expected_reports,
|
||||
"missing_reports": sorted(set(expected_reports) - set(reports)),
|
||||
}
|
||||
Path(manifest_path).write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
}
|
||||
|
||||
accumulate_exit_status() {
|
||||
local -n aggregate="$1"
|
||||
local child_status="$2"
|
||||
if [[ "$child_status" -gt 1 ]]; then
|
||||
aggregate=2
|
||||
elif [[ "$child_status" -eq 1 && "$aggregate" -eq 0 ]]; then
|
||||
aggregate=1
|
||||
fi
|
||||
}
|
||||
|
||||
repository_fingerprint() {
|
||||
local repo="$1"
|
||||
(
|
||||
set -e
|
||||
cd "$repo" || exit 1
|
||||
git rev-parse HEAD
|
||||
git status --porcelain=v1 -z
|
||||
git diff --binary --no-ext-diff HEAD --
|
||||
while IFS= read -r -d '' path; do
|
||||
printf '%s\0' "$path"
|
||||
if [[ -L "$path" ]]; then
|
||||
printf 'symlink\0'
|
||||
readlink -- "$path"
|
||||
else
|
||||
sha256sum -- "$path"
|
||||
fi
|
||||
done < <(git ls-files --others --exclude-standard -z)
|
||||
) | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
write_workspace_state() {
|
||||
local destination="$1"
|
||||
local head
|
||||
local fingerprint
|
||||
: > "$destination"
|
||||
for repo in "${REPOS[@]}"; do
|
||||
[[ "$index" -gt 0 ]] && printf ',\n'
|
||||
printf ' "%s"' "$repo"
|
||||
index=$((index + 1))
|
||||
if ! head="$(git -C "$repo" rev-parse HEAD)"; then
|
||||
return 2
|
||||
fi
|
||||
if ! fingerprint="$(repository_fingerprint "$repo")"; then
|
||||
return 2
|
||||
fi
|
||||
printf '%s\t%s\t%s\n' \
|
||||
"$repo" \
|
||||
"$head" \
|
||||
"$fingerprint" \
|
||||
>> "$destination"
|
||||
done
|
||||
printf '\n ]\n'
|
||||
printf '}\n'
|
||||
} > "$REPORTS_DIR/manifest.json"
|
||||
}
|
||||
|
||||
write_tool_versions() {
|
||||
local destination="$REPORTS_DIR/tool-versions.txt"
|
||||
local status=0
|
||||
local version
|
||||
: > "$destination"
|
||||
local tool
|
||||
for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do
|
||||
has_tool "$tool" || continue
|
||||
if [[ "$tool" == "gitleaks" ]]; then
|
||||
version="$("$tool" version 2>&1)"
|
||||
else
|
||||
version="$("$tool" --version 2>&1)"
|
||||
fi
|
||||
local version_status=$?
|
||||
version="${version%%$'\n'*}"
|
||||
if [[ "$version_status" -ne 0 || -z "$version" ]]; then
|
||||
printf '%s\t%s\n' "$tool" "version-command-failed:$version_status" >> "$destination"
|
||||
status=2
|
||||
else
|
||||
printf '%s\t%s\n' "$tool" "$version" >> "$destination"
|
||||
fi
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
|
||||
write_report_checksums() {
|
||||
local path
|
||||
local relative_path
|
||||
: > "$REPORTS_DIR/report-sha256.txt"
|
||||
while IFS= read -r path; do
|
||||
[[ -f "$path" ]] || continue
|
||||
relative_path="${path#"$REPORTS_DIR"/}"
|
||||
(cd "$REPORTS_DIR" && sha256sum -- "$relative_path") \
|
||||
>> "$REPORTS_DIR/report-sha256.txt" || return 2
|
||||
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
||||
}
|
||||
|
||||
validate_machine_reports() {
|
||||
python3 - "$REPORTS_DIR" "${!MACHINE_REPORTS[@]}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
root = Path(sys.argv[1]).resolve()
|
||||
errors: list[str] = []
|
||||
validated = 0
|
||||
for raw_path in sorted(set(sys.argv[2:])):
|
||||
path = Path(raw_path)
|
||||
try:
|
||||
relative_path = path.resolve().relative_to(root)
|
||||
except (OSError, ValueError) as exc:
|
||||
errors.append(f"{path}: invalid report path: {exc}")
|
||||
continue
|
||||
if not path.is_file():
|
||||
errors.append(f"{relative_path}: expected report was not produced")
|
||||
continue
|
||||
if path.stat().st_size == 0:
|
||||
errors.append(f"{relative_path}: report is empty")
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
errors.append(f"{relative_path}: invalid JSON: {exc}")
|
||||
continue
|
||||
validated += 1
|
||||
if path.suffix == ".sarif" and (
|
||||
not isinstance(payload, dict)
|
||||
or not isinstance(payload.get("version"), str)
|
||||
or not isinstance(payload.get("runs"), list)
|
||||
):
|
||||
errors.append(f"{relative_path}: invalid SARIF document")
|
||||
continue
|
||||
report_name = path.name
|
||||
if report_name.startswith("bandit") and (
|
||||
not isinstance(payload, dict)
|
||||
or not isinstance(payload.get("errors"), list)
|
||||
or not isinstance(payload.get("metrics"), dict)
|
||||
or not isinstance(payload.get("results"), list)
|
||||
):
|
||||
errors.append(f"{relative_path}: invalid Bandit report")
|
||||
elif report_name.startswith("ruff-security") and not isinstance(payload, list):
|
||||
errors.append(f"{relative_path}: invalid Ruff report")
|
||||
elif report_name.startswith("gitleaks-") and not isinstance(payload, list):
|
||||
errors.append(f"{relative_path}: invalid Gitleaks report")
|
||||
elif report_name.startswith("pip-audit-") and (
|
||||
not isinstance(payload, dict)
|
||||
or not isinstance(payload.get("dependencies"), list)
|
||||
):
|
||||
errors.append(f"{relative_path}: invalid pip-audit report")
|
||||
elif report_name.startswith("npm-audit-") and (
|
||||
not isinstance(payload, dict)
|
||||
or payload.get("error")
|
||||
or not isinstance(payload.get("metadata"), dict)
|
||||
or not isinstance(payload.get("vulnerabilities"), dict)
|
||||
):
|
||||
errors.append(f"{relative_path}: npm audit returned an invalid or error report")
|
||||
elif report_name.startswith("osv-scanner-") and (
|
||||
not isinstance(payload, dict)
|
||||
or "results" not in payload
|
||||
or (payload["results"] is not None and not isinstance(payload["results"], list))
|
||||
):
|
||||
errors.append(f"{relative_path}: invalid OSV-Scanner report")
|
||||
elif report_name == "jscpd-report.json" and (
|
||||
not isinstance(payload, dict)
|
||||
or not isinstance(payload.get("duplicates"), list)
|
||||
or not isinstance(payload.get("statistics"), dict)
|
||||
):
|
||||
errors.append(f"{relative_path}: invalid jscpd report")
|
||||
|
||||
if errors:
|
||||
print("\n".join(errors), file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
print(f"Validated {validated} JSON/SARIF reports.")
|
||||
PY
|
||||
}
|
||||
|
||||
run_semgrep() {
|
||||
local report="$REPORTS_DIR/semgrep.sarif"
|
||||
prepare_machine_report "$report" || return 2
|
||||
local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml")
|
||||
if [[ "$MODE" != "quick" ]]; then
|
||||
config_args+=(--config p/default --config p/owasp-top-ten)
|
||||
fi
|
||||
semgrep scan \
|
||||
--metrics=off \
|
||||
--error \
|
||||
--no-git-ignore \
|
||||
--sarif \
|
||||
--output "$report" \
|
||||
@@ -198,10 +490,15 @@ run_semgrep() {
|
||||
run_bandit() {
|
||||
local status=0
|
||||
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
||||
bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" || status=1
|
||||
prepare_machine_report "$REPORTS_DIR/bandit.json" || return 2
|
||||
bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json"
|
||||
accumulate_exit_status status "$?"
|
||||
fi
|
||||
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
||||
bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" || true
|
||||
prepare_machine_report "$REPORTS_DIR/bandit-tests.json" || return 2
|
||||
bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json"
|
||||
local test_status=$?
|
||||
[[ "$test_status" -le 1 ]] || status=2
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
@@ -209,10 +506,15 @@ run_bandit() {
|
||||
run_ruff_security() {
|
||||
local status=0
|
||||
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
||||
ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" || status=1
|
||||
prepare_machine_report "$REPORTS_DIR/ruff-security.json" || return 2
|
||||
ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json"
|
||||
accumulate_exit_status status "$?"
|
||||
fi
|
||||
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
||||
ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" || true
|
||||
prepare_machine_report "$REPORTS_DIR/ruff-security-tests.json" || return 2
|
||||
ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json"
|
||||
local test_status=$?
|
||||
[[ "$test_status" -le 1 ]] || status=2
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
@@ -223,12 +525,15 @@ run_gitleaks() {
|
||||
local name
|
||||
name="$(safe_name "$repo")"
|
||||
if gitleaks git --help >/dev/null 2>&1; then
|
||||
prepare_machine_report "$REPORTS_DIR/gitleaks-history-$name.json" || return 2
|
||||
prepare_machine_report "$REPORTS_DIR/gitleaks-worktree-$name.json" || return 2
|
||||
gitleaks git \
|
||||
--config "$ROOT/.gitleaks.toml" \
|
||||
--report-format json \
|
||||
--report-path "$REPORTS_DIR/gitleaks-history-$name.json" \
|
||||
--no-banner \
|
||||
"$repo" || status=1
|
||||
"$repo"
|
||||
accumulate_exit_status status "$?"
|
||||
# History scanning does not include new or modified working-tree files.
|
||||
# Scan the directory as well so pre-commit audits cover the exact code
|
||||
# under review, while retaining the history scan above.
|
||||
@@ -237,14 +542,17 @@ run_gitleaks() {
|
||||
--report-format json \
|
||||
--report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \
|
||||
--no-banner \
|
||||
"$repo" || status=1
|
||||
"$repo"
|
||||
accumulate_exit_status status "$?"
|
||||
else
|
||||
prepare_machine_report "$REPORTS_DIR/gitleaks-$name.json" || return 2
|
||||
gitleaks detect \
|
||||
--source "$repo" \
|
||||
--config "$ROOT/.gitleaks.toml" \
|
||||
--report-format json \
|
||||
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
|
||||
--no-banner || status=1
|
||||
--no-banner
|
||||
accumulate_exit_status status "$?"
|
||||
fi
|
||||
done
|
||||
return "$status"
|
||||
@@ -255,6 +563,7 @@ run_trivy() {
|
||||
for repo in "${REPOS[@]}"; do
|
||||
local name
|
||||
name="$(safe_name "$repo")"
|
||||
prepare_machine_report "$REPORTS_DIR/trivy-$name.sarif" || return 2
|
||||
trivy fs \
|
||||
--scanners vuln,secret,misconfig \
|
||||
--skip-dirs node_modules \
|
||||
@@ -262,8 +571,10 @@ run_trivy() {
|
||||
--skip-dirs dist \
|
||||
--skip-dirs build \
|
||||
--format sarif \
|
||||
--exit-code 1 \
|
||||
--output "$REPORTS_DIR/trivy-$name.sarif" \
|
||||
"$repo" || status=1
|
||||
"$repo"
|
||||
accumulate_exit_status status "$?"
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
@@ -274,9 +585,11 @@ run_pip_audit_manifests() {
|
||||
[[ -f "$repo/requirements.txt" ]] || continue
|
||||
local name
|
||||
name="$(safe_name "$repo")"
|
||||
prepare_machine_report "$REPORTS_DIR/pip-audit-$name.json" || return 2
|
||||
# Requirements may contain project-relative entries such as `.[server]`.
|
||||
# Resolve them from the owning repository instead of the meta-repository.
|
||||
(cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json") || status=1
|
||||
(cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json")
|
||||
accumulate_exit_status status "$?"
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
@@ -287,7 +600,9 @@ run_npm_audit_manifests() {
|
||||
[[ -f "$repo/webui/package-lock.json" ]] || continue
|
||||
local name
|
||||
name="$(safe_name "$repo")"
|
||||
(cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") || status=1
|
||||
prepare_machine_report "$REPORTS_DIR/npm-audit-$name.json" || return 2
|
||||
(cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json")
|
||||
accumulate_exit_status status "$?"
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
@@ -298,19 +613,24 @@ run_osv_scanner() {
|
||||
local name
|
||||
name="$(safe_name "$repo")"
|
||||
if osv-scanner scan --help >/dev/null 2>&1; then
|
||||
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
||||
osv-scanner scan -r \
|
||||
--allow-no-lockfiles \
|
||||
--format json \
|
||||
--output-file "$REPORTS_DIR/osv-scanner-$name.json" \
|
||||
"$repo" || status=1
|
||||
"$repo"
|
||||
accumulate_exit_status status "$?"
|
||||
else
|
||||
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" || status=1
|
||||
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
||||
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo"
|
||||
accumulate_exit_status status "$?"
|
||||
fi
|
||||
done
|
||||
return "$status"
|
||||
}
|
||||
|
||||
run_jscpd() {
|
||||
prepare_machine_report "$REPORTS_DIR/jscpd/jscpd-report.json" || return 2
|
||||
local ignored_paths
|
||||
ignored_paths=(
|
||||
"**/.git/**"
|
||||
@@ -349,55 +669,66 @@ run_jscpd() {
|
||||
|
||||
run_radon() {
|
||||
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
||||
register_report "$REPORTS_DIR/radon-cc.txt" || return 2
|
||||
radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt"
|
||||
}
|
||||
|
||||
run_xenon() {
|
||||
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
||||
register_report "$REPORTS_DIR/xenon.txt" || return 2
|
||||
xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1
|
||||
}
|
||||
|
||||
write_manifest
|
||||
register_report "$REPORTS_DIR/step-status.tsv"
|
||||
register_report "$REPORTS_DIR/tool-versions.txt"
|
||||
register_report "$REPORTS_DIR/workspace-state-start.tsv"
|
||||
register_report "$REPORTS_DIR/workspace-state-end.tsv"
|
||||
: > "$REPORTS_DIR/step-status.tsv"
|
||||
if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then
|
||||
echo "Could not capture the repository state before the audit." >&2
|
||||
exit 1
|
||||
fi
|
||||
run_step "Record audit tool versions" execution-only write_tool_versions
|
||||
|
||||
if has_tool semgrep; then
|
||||
run_step "Semgrep SAST" run_semgrep
|
||||
run_step "Semgrep SAST" findings-exit-one run_semgrep
|
||||
else
|
||||
skip_or_fail_missing semgrep
|
||||
fi
|
||||
|
||||
if has_tool bandit; then
|
||||
run_step "Bandit Python security scan" run_bandit
|
||||
run_step "Bandit Python security scan" findings-exit-one run_bandit
|
||||
else
|
||||
skip_or_fail_missing bandit
|
||||
fi
|
||||
|
||||
if has_tool ruff; then
|
||||
run_step "Ruff flake8-bandit security rules" run_ruff_security
|
||||
run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
|
||||
else
|
||||
skip_or_fail_missing ruff
|
||||
fi
|
||||
|
||||
if has_tool gitleaks; then
|
||||
run_step "Gitleaks secret scan" run_gitleaks
|
||||
run_step "Gitleaks secret scan" findings-exit-one run_gitleaks
|
||||
else
|
||||
skip_or_fail_missing gitleaks
|
||||
fi
|
||||
|
||||
if [[ "$MODE" != "quick" ]]; then
|
||||
if has_tool trivy; then
|
||||
run_step "Trivy filesystem vulnerability/secret/misconfig scan" run_trivy
|
||||
run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
|
||||
else
|
||||
skip_or_fail_missing trivy
|
||||
fi
|
||||
|
||||
if has_tool pip-audit; then
|
||||
run_step "pip-audit requirements scan" run_pip_audit_manifests
|
||||
run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
|
||||
else
|
||||
skip_or_fail_missing pip-audit
|
||||
fi
|
||||
|
||||
if has_tool npm; then
|
||||
run_step "npm audit lockfile scan" run_npm_audit_manifests
|
||||
run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
|
||||
else
|
||||
skip_or_fail_missing npm
|
||||
fi
|
||||
@@ -405,37 +736,62 @@ fi
|
||||
|
||||
if [[ "$MODE" == "full" ]]; then
|
||||
if has_tool osv-scanner; then
|
||||
run_step "OSV-Scanner recursive dependency scan" run_osv_scanner
|
||||
run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
|
||||
else
|
||||
skip_or_fail_missing osv-scanner
|
||||
fi
|
||||
|
||||
if has_tool jscpd; then
|
||||
run_step "jscpd duplicate code scan" run_jscpd
|
||||
run_step "jscpd duplicate code scan" execution-only run_jscpd
|
||||
else
|
||||
skip_or_fail_missing jscpd
|
||||
fi
|
||||
|
||||
if has_tool radon; then
|
||||
run_step "Radon complexity report" run_radon
|
||||
run_step "Radon complexity report" execution-only run_radon
|
||||
else
|
||||
skip_or_fail_missing radon
|
||||
fi
|
||||
|
||||
if has_tool xenon; then
|
||||
run_step "Xenon complexity threshold scan" run_xenon
|
||||
run_step "Xenon complexity threshold scan" findings-exit-one run_xenon
|
||||
else
|
||||
skip_or_fail_missing xenon
|
||||
fi
|
||||
fi
|
||||
|
||||
run_step "Validate machine-readable audit reports" execution-only validate_machine_reports
|
||||
|
||||
echo
|
||||
echo "Security audit reports written to: $REPORTS_DIR"
|
||||
if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then
|
||||
echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings."
|
||||
fi
|
||||
if [[ "$missing_status" -ne 0 ]]; then
|
||||
echo "One or more required tools were missing." >&2
|
||||
echo "One or more audit tools were missing." >&2
|
||||
fi
|
||||
|
||||
if ! write_workspace_state "$REPORTS_DIR/workspace-state-end.tsv"; then
|
||||
workspace_unchanged=false
|
||||
execution_status=1
|
||||
overall_status=1
|
||||
echo "Could not capture the repository state after the audit." >&2
|
||||
elif cmp -s "$REPORTS_DIR/workspace-state-start.tsv" "$REPORTS_DIR/workspace-state-end.tsv"; then
|
||||
workspace_unchanged=true
|
||||
else
|
||||
workspace_unchanged=false
|
||||
overall_status=1
|
||||
echo "Repository state changed during the audit; discard this mixed snapshot and rerun." >&2
|
||||
fi
|
||||
audit_completed_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
if ! write_report_checksums; then
|
||||
execution_status=1
|
||||
overall_status=1
|
||||
echo "Could not generate report checksums." >&2
|
||||
fi
|
||||
if ! write_manifest; then
|
||||
echo "Could not write the audit manifest." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit "$overall_status"
|
||||
|
||||
130
tools/checks/check-version-alignment.py
Normal file
130
tools/checks/check-version-alignment.py
Normal file
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail when backend, manifest, frontend, lock, or release-ref versions drift."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import asdict
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||
|
||||
from govoplan_release.version_alignment import ( # noqa: E402
|
||||
release_composition_issues,
|
||||
repository_version_issues,
|
||||
selected_repository_version_issues,
|
||||
)
|
||||
from govoplan_release.workspace import ( # noqa: E402
|
||||
load_repository_specs,
|
||||
resolve_repo_path,
|
||||
resolve_workspace_root,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--workspace-root", type=Path, default=None)
|
||||
parser.add_argument("--repo", action="append", default=[], help="Repository name to check; may be repeated.")
|
||||
parser.add_argument(
|
||||
"--repo-version",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="REPO=VERSION",
|
||||
help="Require a repository's aligned source metadata to equal VERSION; may be repeated.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--release-composition",
|
||||
action="store_true",
|
||||
help="Also compare tagged backend and WebUI references in the release composition.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-metadata-only",
|
||||
action="store_true",
|
||||
help="Skip lockfile checks for the pre-commit source gate; never use for publication.",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Print machine-readable output.")
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = resolve_workspace_root(args.workspace_root)
|
||||
expected_versions = _parse_repo_versions(parser, args.repo_version)
|
||||
selected = {*args.repo, *expected_versions}
|
||||
known = {spec.name for spec in load_repository_specs()}
|
||||
unknown = sorted(selected - known)
|
||||
if unknown:
|
||||
parser.error(f"unknown repository name(s): {', '.join(unknown)}")
|
||||
|
||||
checked: list[str] = []
|
||||
issues = list(
|
||||
selected_repository_version_issues(
|
||||
repo_versions=expected_versions,
|
||||
workspace=workspace_root,
|
||||
include_lockfiles=not args.source_metadata_only,
|
||||
)
|
||||
)
|
||||
for spec in load_repository_specs():
|
||||
if selected and spec.name not in selected:
|
||||
continue
|
||||
repo_path = resolve_repo_path(spec, workspace_root)
|
||||
if not repo_path.exists():
|
||||
continue
|
||||
checked.append(spec.name)
|
||||
if spec.name in expected_versions:
|
||||
continue
|
||||
issues.extend(
|
||||
repository_version_issues(
|
||||
repo_path,
|
||||
include_lockfiles=not args.source_metadata_only,
|
||||
)
|
||||
)
|
||||
|
||||
if args.release_composition:
|
||||
issues.extend(
|
||||
release_composition_issues(
|
||||
META_ROOT,
|
||||
core_root=workspace_root / "govoplan-core",
|
||||
)
|
||||
)
|
||||
|
||||
payload = {
|
||||
"workspace_root": str(workspace_root),
|
||||
"repositories_checked": checked,
|
||||
"release_composition_checked": args.release_composition,
|
||||
"issues": [asdict(issue) for issue in issues],
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
elif issues:
|
||||
print("Version alignment failed:", file=sys.stderr)
|
||||
for issue in issues:
|
||||
print(
|
||||
f"- {issue.repo}: {issue.source}: {issue.actual!r}; expected {issue.expected!r} "
|
||||
f"({issue.message})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
suffix = " including release composition" if args.release_composition else ""
|
||||
print(f"Version alignment passed for {len(checked)} repositories{suffix}.")
|
||||
return 1 if issues else 0
|
||||
|
||||
|
||||
def _parse_repo_versions(parser: argparse.ArgumentParser, values: list[str]) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for value in values:
|
||||
if "=" not in value:
|
||||
parser.error(f"--repo-version must use REPO=VERSION: {value}")
|
||||
repo, version = (item.strip() for item in value.split("=", 1))
|
||||
version = version.removeprefix("v")
|
||||
if not repo or not version:
|
||||
parser.error(f"--repo-version must use REPO=VERSION: {value}")
|
||||
if repo in result and result[repo] != version:
|
||||
parser.error(f"conflicting versions requested for {repo}")
|
||||
result[repo] = version
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
# Application imports intentionally follow the test environment and source-path setup below.
|
||||
# ruff: noqa: E402
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
@@ -31,7 +34,6 @@ from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mod
|
||||
from govoplan_core.core.module_installer import (
|
||||
cancel_module_installer_request,
|
||||
claim_next_module_installer_request,
|
||||
module_installer_daemon_status,
|
||||
module_installer_lock_status,
|
||||
queue_module_installer_request,
|
||||
read_module_installer_run,
|
||||
@@ -44,8 +46,6 @@ from govoplan_core.core.module_installer import (
|
||||
)
|
||||
from govoplan_core.core.module_management import (
|
||||
ModuleInstallPlan,
|
||||
ModuleInstallPlanItem,
|
||||
saved_desired_enabled_modules,
|
||||
saved_module_install_plan,
|
||||
save_desired_enabled_modules,
|
||||
save_module_install_plan,
|
||||
|
||||
380
tools/checks/release_integration.py
Normal file
380
tools/checks/release_integration.py
Normal file
@@ -0,0 +1,380 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.metadata as metadata
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CORE_RELEASE_TEST_MODULES = (
|
||||
"tests.test_module_system",
|
||||
"tests.test_access_contracts",
|
||||
"tests.test_identity_organization_contracts",
|
||||
"tests.test_core_events",
|
||||
"tests.test_policy_contracts",
|
||||
)
|
||||
|
||||
# These tests intentionally compare the current source workspace with sibling
|
||||
# source repositories. The release gate instead installs independently tagged
|
||||
# wheels and verifies the equivalent contracts below from artifact metadata and
|
||||
# public module capability factories.
|
||||
SOURCE_COUPLED_CORE_TESTS: Mapping[str, str] = {
|
||||
"tests.test_module_system.ModuleSystemTests.test_discovers_installed_core_and_product_module_manifests": (
|
||||
"asserts the current Campaign source manifest rather than the independently tagged artifact"
|
||||
),
|
||||
"tests.test_module_system.ModuleSystemTests.test_module_manifest_versions_match_source_project_versions": (
|
||||
"requires sibling source pyproject.toml files that wheels do not contain"
|
||||
),
|
||||
"tests.test_module_system.ModuleSystemTests.test_release_catalog_generator_reads_manifest_interface_contracts": (
|
||||
"requires the sibling meta/source workspace and is retained by the normal source gate"
|
||||
),
|
||||
"tests.test_identity_organization_contracts.IdentityOrganizationContractTests."
|
||||
"test_sql_identity_and_organization_directories_return_dtos": (
|
||||
"directly constructs the current IDM source implementation instead of using its public capability factory"
|
||||
),
|
||||
}
|
||||
|
||||
_RELEASE_REQUIREMENT_PATTERN = re.compile(
|
||||
r"^(govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://.+?\.git@v[0-9]+\.[0-9]+\.[0-9]+$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledModuleArtifact:
|
||||
package_name: str
|
||||
package_version: str
|
||||
manifest_id: str
|
||||
manifest_version: str
|
||||
|
||||
|
||||
def release_package_names(path: Path) -> tuple[str, ...]:
|
||||
names: list[str] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
match = _RELEASE_REQUIREMENT_PATTERN.match(line.strip())
|
||||
if match:
|
||||
names.append(match.group(1))
|
||||
return tuple(dict.fromkeys(names))
|
||||
|
||||
|
||||
def artifact_contract_issues(
|
||||
artifacts: Sequence[InstalledModuleArtifact],
|
||||
*,
|
||||
release_packages: Sequence[str],
|
||||
available_manifest_versions: Mapping[str, str],
|
||||
) -> tuple[str, ...]:
|
||||
issues: list[str] = []
|
||||
artifacts_by_package: dict[str, list[InstalledModuleArtifact]] = {}
|
||||
artifacts_by_module: dict[str, list[InstalledModuleArtifact]] = {}
|
||||
for artifact in artifacts:
|
||||
artifacts_by_package.setdefault(artifact.package_name, []).append(artifact)
|
||||
artifacts_by_module.setdefault(artifact.manifest_id, []).append(artifact)
|
||||
if artifact.package_version != artifact.manifest_version:
|
||||
issues.append(
|
||||
f"{artifact.package_name} metadata version {artifact.package_version!r} does not match "
|
||||
f"manifest {artifact.manifest_id!r} version {artifact.manifest_version!r}"
|
||||
)
|
||||
available_version = available_manifest_versions.get(artifact.manifest_id)
|
||||
if available_version is None:
|
||||
issues.append(f"Installed manifest {artifact.manifest_id!r} is not discoverable through Core")
|
||||
elif available_version != artifact.manifest_version:
|
||||
issues.append(
|
||||
f"Discovered manifest {artifact.manifest_id!r} version {available_version!r} does not match "
|
||||
f"its {artifact.package_name} entry point version {artifact.manifest_version!r}"
|
||||
)
|
||||
|
||||
for package_name in release_packages:
|
||||
if package_name not in artifacts_by_package:
|
||||
issues.append(f"{package_name} does not publish a govoplan.modules entry point")
|
||||
for module_id, providers in artifacts_by_module.items():
|
||||
if len(providers) > 1:
|
||||
packages = ", ".join(sorted(item.package_name for item in providers))
|
||||
issues.append(f"Module id {module_id!r} is published by multiple release packages: {packages}")
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def iter_test_cases(suite: unittest.TestSuite) -> Iterator[unittest.TestCase]:
|
||||
for test in suite:
|
||||
if isinstance(test, unittest.TestSuite):
|
||||
yield from iter_test_cases(test)
|
||||
else:
|
||||
yield test
|
||||
|
||||
|
||||
def filter_test_suite(
|
||||
suite: unittest.TestSuite,
|
||||
excluded_ids: Sequence[str],
|
||||
) -> tuple[unittest.TestSuite, tuple[str, ...]]:
|
||||
excluded = set(excluded_ids)
|
||||
kept: list[unittest.TestCase] = []
|
||||
removed: list[str] = []
|
||||
for test in iter_test_cases(suite):
|
||||
test_id = test.id()
|
||||
if test_id in excluded:
|
||||
removed.append(test_id)
|
||||
else:
|
||||
kept.append(test)
|
||||
return unittest.TestSuite(kept), tuple(removed)
|
||||
|
||||
|
||||
def _collect_installed_artifacts(
|
||||
package_names: Sequence[str],
|
||||
) -> tuple[tuple[InstalledModuleArtifact, ...], tuple[str, ...]]:
|
||||
artifacts: list[InstalledModuleArtifact] = []
|
||||
issues: list[str] = []
|
||||
for package_name in package_names:
|
||||
try:
|
||||
distribution = metadata.distribution(package_name)
|
||||
except metadata.PackageNotFoundError:
|
||||
issues.append(f"Release package is not installed: {package_name}")
|
||||
continue
|
||||
entry_points = tuple(
|
||||
entry_point
|
||||
for entry_point in distribution.entry_points
|
||||
if entry_point.group == "govoplan.modules"
|
||||
)
|
||||
if not entry_points:
|
||||
continue
|
||||
for entry_point in entry_points:
|
||||
try:
|
||||
factory = entry_point.load()
|
||||
manifest = factory()
|
||||
manifest_id = str(manifest.id)
|
||||
manifest_version = str(manifest.version)
|
||||
except Exception as exc:
|
||||
issues.append(f"Could not load {package_name} module entry point {entry_point.name!r}: {exc}")
|
||||
continue
|
||||
artifacts.append(
|
||||
InstalledModuleArtifact(
|
||||
package_name=package_name,
|
||||
package_version=distribution.version,
|
||||
manifest_id=manifest_id,
|
||||
manifest_version=manifest_version,
|
||||
)
|
||||
)
|
||||
return tuple(artifacts), tuple(issues)
|
||||
|
||||
|
||||
def _require_equal(actual: object, expected: object, message: str) -> None:
|
||||
if actual != expected:
|
||||
raise RuntimeError(f"{message}: expected {expected!r}, got {actual!r}")
|
||||
|
||||
|
||||
def _run_directory_capability_smoke(registry: object) -> None:
|
||||
from govoplan_core.core.identity import CAPABILITY_IDENTITY_DIRECTORY, IdentityDirectory
|
||||
from govoplan_core.core.idm import CAPABILITY_IDM_DIRECTORY, IdmDirectory
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_core.core.organizations import CAPABILITY_ORGANIZATION_DIRECTORY, OrganizationDirectory
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.db.session import configure_database, reset_database, set_database
|
||||
from govoplan_core.tenancy.scope import create_scope_tables
|
||||
from govoplan_identity.backend.db.models import Identity, IdentityAccountLink
|
||||
from govoplan_idm.backend.db.models import IdmOrganizationFunctionAssignment
|
||||
from govoplan_organizations.backend.db.models import (
|
||||
OrganizationFunction,
|
||||
OrganizationFunctionType,
|
||||
OrganizationUnit,
|
||||
OrganizationUnitType,
|
||||
)
|
||||
from govoplan_tenancy.backend.db.models import Tenant
|
||||
|
||||
registry.configure_capability_context(ModuleContext(registry=registry, settings=object()))
|
||||
identity_directory = registry.require_capability(CAPABILITY_IDENTITY_DIRECTORY)
|
||||
organization_directory = registry.require_capability(CAPABILITY_ORGANIZATION_DIRECTORY)
|
||||
idm_directory = registry.require_capability(CAPABILITY_IDM_DIRECTORY)
|
||||
if not isinstance(identity_directory, IdentityDirectory):
|
||||
raise RuntimeError("Installed Identity capability does not satisfy IdentityDirectory")
|
||||
if not isinstance(organization_directory, OrganizationDirectory):
|
||||
raise RuntimeError("Installed Organizations capability does not satisfy OrganizationDirectory")
|
||||
if not isinstance(idm_directory, IdmDirectory):
|
||||
raise RuntimeError("Installed IDM capability does not satisfy IdmDirectory")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-release-directory-") as temp_dir:
|
||||
database = configure_database(
|
||||
f"sqlite:///{Path(temp_dir) / 'directory.sqlite3'}",
|
||||
dispose_previous=True,
|
||||
)
|
||||
set_database(database)
|
||||
try:
|
||||
create_scope_tables(database.engine)
|
||||
Base.metadata.create_all(bind=database.engine)
|
||||
with database.session() as session:
|
||||
tenant = Tenant(
|
||||
id="release-tenant",
|
||||
slug="release-tenant",
|
||||
name="Release Tenant",
|
||||
settings={},
|
||||
)
|
||||
identity = Identity(
|
||||
id="release-identity",
|
||||
display_name="Release Person",
|
||||
source="local",
|
||||
)
|
||||
link = IdentityAccountLink(
|
||||
identity_id=identity.id,
|
||||
account_id="release-account",
|
||||
is_primary=True,
|
||||
)
|
||||
unit_type = OrganizationUnitType(
|
||||
id="release-unit-type",
|
||||
tenant_id=tenant.id,
|
||||
slug="office",
|
||||
name="Office",
|
||||
)
|
||||
unit = OrganizationUnit(
|
||||
id="release-unit",
|
||||
tenant_id=tenant.id,
|
||||
unit_type_id=unit_type.id,
|
||||
slug="registry",
|
||||
name="Registry",
|
||||
)
|
||||
function_type = OrganizationFunctionType(
|
||||
id="release-function-type",
|
||||
tenant_id=tenant.id,
|
||||
slug="clerk",
|
||||
name="Clerk",
|
||||
organization_unit_type_id=unit_type.id,
|
||||
)
|
||||
function = OrganizationFunction(
|
||||
id="release-function",
|
||||
tenant_id=tenant.id,
|
||||
function_type_id=function_type.id,
|
||||
organization_unit_id=unit.id,
|
||||
slug="registry-clerk",
|
||||
name="Registry Clerk",
|
||||
)
|
||||
assignment = IdmOrganizationFunctionAssignment(
|
||||
id="release-assignment",
|
||||
tenant_id=tenant.id,
|
||||
identity_id=identity.id,
|
||||
function_id=function.id,
|
||||
organization_unit_id=unit.id,
|
||||
applies_to_subunits=True,
|
||||
)
|
||||
session.add_all((tenant, identity, link, unit_type, unit, function_type, function, assignment))
|
||||
session.commit()
|
||||
|
||||
identity_ref = identity_directory.identity_for_account("release-account")
|
||||
_require_equal(
|
||||
identity_ref.id if identity_ref is not None else None,
|
||||
"release-identity",
|
||||
"Identity capability account projection",
|
||||
)
|
||||
function_ref = organization_directory.get_function("release-function")
|
||||
_require_equal(
|
||||
function_ref.name if function_ref is not None else None,
|
||||
"Registry Clerk",
|
||||
"Organizations capability function projection",
|
||||
)
|
||||
identity_assignments = idm_directory.organization_function_assignments_for_identity(
|
||||
"release-identity",
|
||||
tenant_id="release-tenant",
|
||||
)
|
||||
account_assignments = idm_directory.organization_function_assignments_for_account(
|
||||
"release-account",
|
||||
tenant_id="release-tenant",
|
||||
)
|
||||
_require_equal(
|
||||
tuple(item.id for item in identity_assignments),
|
||||
("release-assignment",),
|
||||
"IDM identity assignment projection",
|
||||
)
|
||||
_require_equal(
|
||||
tuple(item.id for item in account_assignments),
|
||||
("release-assignment",),
|
||||
"IDM account assignment projection",
|
||||
)
|
||||
finally:
|
||||
reset_database(dispose=True)
|
||||
|
||||
|
||||
def run_installed_artifact_checks(requirements_path: Path) -> int:
|
||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||
|
||||
package_names = release_package_names(requirements_path)
|
||||
if not package_names:
|
||||
print(f"No tagged GovOPlaN packages found in {requirements_path}", file=sys.stderr)
|
||||
return 1
|
||||
artifacts, collection_issues = _collect_installed_artifacts(package_names)
|
||||
available = available_module_manifests()
|
||||
contract_issues = artifact_contract_issues(
|
||||
artifacts,
|
||||
release_packages=package_names,
|
||||
available_manifest_versions={module_id: str(manifest.version) for module_id, manifest in available.items()},
|
||||
)
|
||||
issues = (*collection_issues, *contract_issues)
|
||||
if issues:
|
||||
print("\n".join(issues), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
module_ids = tuple(dict.fromkeys(artifact.manifest_id for artifact in artifacts))
|
||||
registry = build_platform_registry(module_ids)
|
||||
snapshot = registry.validate()
|
||||
discovered_ids = {manifest.id for manifest in snapshot.manifests}
|
||||
if discovered_ids != set(module_ids):
|
||||
print(
|
||||
"Release registry module mismatch: "
|
||||
f"expected {', '.join(sorted(module_ids))}; got {', '.join(sorted(discovered_ids))}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
_run_directory_capability_smoke(registry)
|
||||
versions = ", ".join(
|
||||
f"{artifact.manifest_id}={artifact.manifest_version}"
|
||||
for artifact in sorted(artifacts, key=lambda item: item.manifest_id)
|
||||
)
|
||||
print(f"Installed release artifact contracts passed: {versions}")
|
||||
return 0
|
||||
|
||||
|
||||
def run_core_release_tests(core_root: Path) -> int:
|
||||
resolved_root = core_root.resolve()
|
||||
os.chdir(resolved_root)
|
||||
sys.path.insert(0, str(resolved_root))
|
||||
loader = unittest.defaultTestLoader
|
||||
loaded = loader.loadTestsFromNames(CORE_RELEASE_TEST_MODULES)
|
||||
suite, removed = filter_test_suite(loaded, tuple(SOURCE_COUPLED_CORE_TESTS))
|
||||
missing_exclusions = set(SOURCE_COUPLED_CORE_TESTS) - set(removed)
|
||||
if missing_exclusions:
|
||||
print(
|
||||
"Release source-test exclusions no longer match the Core suite: "
|
||||
+ ", ".join(sorted(missing_exclusions)),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
for test_id in removed:
|
||||
print(f"Artifact contract replaces source-coupled test: {test_id} ({SOURCE_COUPLED_CORE_TESTS[test_id]})")
|
||||
result = unittest.TextTestRunner(verbosity=1).run(suite)
|
||||
return 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
meta_root = Path(__file__).resolve().parents[2]
|
||||
parser = argparse.ArgumentParser(description="Artifact-aware GovOPlaN release integration checks")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
artifact_parser = subparsers.add_parser("artifacts", help="Validate installed module artifacts and capabilities")
|
||||
artifact_parser.add_argument(
|
||||
"--requirements",
|
||||
type=Path,
|
||||
default=meta_root / "requirements-release.txt",
|
||||
)
|
||||
core_parser = subparsers.add_parser("core-tests", help="Run Core tests that are valid for tagged module artifacts")
|
||||
core_parser.add_argument("--core-root", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
if args.command == "artifacts":
|
||||
return run_installed_artifact_checks(args.requirements)
|
||||
return run_core_release_tests(args.core_root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -31,19 +31,24 @@ RUN apt-get update \
|
||||
COPY --from=go-tools /go/bin/gitleaks /usr/local/bin/gitleaks
|
||||
COPY --from=go-tools /go/bin/osv-scanner /usr/local/bin/osv-scanner
|
||||
COPY requirements-audit.txt /tmp/requirements-audit.txt
|
||||
COPY tools/checks/security-audit/semgrep-govoplan.yml /tmp/semgrep-govoplan.yml
|
||||
COPY tools/checks/check-version-alignment.py /tmp/semgrep-smoke.py
|
||||
|
||||
RUN python -m pip install --upgrade pip \
|
||||
&& python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \
|
||||
&& python -m pip install --no-cache-dir "semgrep>=1.140,<2" \
|
||||
&& python -m pip install --no-cache-dir --upgrade --no-deps "click>=8.3.3" \
|
||||
&& python -m pip install --no-cache-dir --upgrade --no-deps "click==8.3.3" \
|
||||
&& python -m pip install --no-cache-dir --upgrade --no-deps "mcp==1.28.1" \
|
||||
&& npm install -g jscpd \
|
||||
&& semgrep --version \
|
||||
&& semgrep scan --metrics=off --config /tmp/semgrep-govoplan.yml /tmp/semgrep-smoke.py \
|
||||
&& bandit --version \
|
||||
&& ruff --version \
|
||||
&& gitleaks version \
|
||||
&& osv-scanner --version \
|
||||
&& trivy --version \
|
||||
&& jscpd --version
|
||||
&& jscpd --version \
|
||||
&& pip-audit --progress-spinner off
|
||||
|
||||
RUN mkdir -p /workspace \
|
||||
&& chmod 0777 /workspace
|
||||
|
||||
@@ -84,6 +84,8 @@ IMAGE_FINGERPRINT="$(
|
||||
{
|
||||
sha256sum "$ROOT/tools/checks/security-audit/Dockerfile"
|
||||
sha256sum "$ROOT/requirements-audit.txt"
|
||||
sha256sum "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml"
|
||||
sha256sum "$ROOT/tools/checks/check-version-alignment.py"
|
||||
} | sha256sum | cut -c1-16
|
||||
)"
|
||||
FINGERPRINT_IMAGE="${SECURITY_AUDIT_FINGERPRINT_IMAGE:-$IMAGE_REPOSITORY:$IMAGE_FINGERPRINT}"
|
||||
@@ -129,6 +131,7 @@ docker run --rm \
|
||||
-e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \
|
||||
-e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \
|
||||
-e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \
|
||||
-e SECURITY_AUDIT_TOOLBOX_FINGERPRINT="$IMAGE_FINGERPRINT" \
|
||||
"${EXTRA_ENV[@]}" \
|
||||
-v "$MOUNT_ROOT:/workspace" \
|
||||
-w "$WORKDIR" \
|
||||
|
||||
@@ -7,7 +7,6 @@ import argparse
|
||||
import base64
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
|
||||
@@ -19,9 +19,11 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
|
||||
sys.path.insert(0, str(CORE_ROOT / "src"))
|
||||
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||
from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402
|
||||
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||
@@ -196,6 +198,19 @@ def main() -> int:
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version.removeprefix("v")
|
||||
version_issues = selected_repository_version_issues(
|
||||
repo_versions={
|
||||
"govoplan-core": version,
|
||||
**{module.repo: version for module in CATALOG_MODULES},
|
||||
},
|
||||
workspace=CORE_ROOT.parent,
|
||||
)
|
||||
if version_issues:
|
||||
details = "; ".join(
|
||||
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
|
||||
for issue in version_issues
|
||||
)
|
||||
parser.error(f"version alignment gate failed: {details}")
|
||||
tag = f"v{version}"
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
|
||||
@@ -17,6 +17,10 @@ the regenerated release lockfile.
|
||||
Options:
|
||||
--npm <path> npm executable to use.
|
||||
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
|
||||
--local-git-repo <path>
|
||||
Resolve that repository's release tag from its local Git
|
||||
object store while preserving the published Git URL in
|
||||
the lockfile. May be repeated.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
@@ -32,6 +36,7 @@ CORE_ROOT="$(cd "$CORE_ROOT" && pwd)"
|
||||
WEBUI="$CORE_ROOT/webui"
|
||||
NPM_BIN="${NPM:-}"
|
||||
NODE_BIN="${NODE:-}"
|
||||
LOCAL_GIT_REPOS=()
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
@@ -61,6 +66,11 @@ while [[ $# -gt 0 ]]; do
|
||||
WEBUI="$CORE_ROOT/webui"
|
||||
shift 2
|
||||
;;
|
||||
--local-git-repo)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
LOCAL_GIT_REPOS+=("$(cd "$2" && pwd)")
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -90,9 +100,27 @@ fi
|
||||
|
||||
echo "Generating release lockfile from $WEBUI/package.release.json"
|
||||
echo "Temporary workspace: $TMP_DIR"
|
||||
|
||||
GIT_ENV=(env)
|
||||
git_config_count=0
|
||||
for local_repo in "${LOCAL_GIT_REPOS[@]}"; do
|
||||
[[ -d "$local_repo/.git" ]] || fail "local Git release source is not a repository: $local_repo"
|
||||
remote_url="$(git -C "$local_repo" remote get-url origin)"
|
||||
remote_url="${remote_url#git+}"
|
||||
if [[ "$remote_url" != *://* && "$remote_url" =~ ^([^@]+@)?([^:]+):(.+)$ ]]; then
|
||||
remote_url="ssh://${BASH_REMATCH[1]}${BASH_REMATCH[2]}/${BASH_REMATCH[3]}"
|
||||
fi
|
||||
GIT_ENV+=(
|
||||
"GIT_CONFIG_KEY_${git_config_count}=url.file://$local_repo/.insteadOf"
|
||||
"GIT_CONFIG_VALUE_${git_config_count}=$remote_url"
|
||||
)
|
||||
git_config_count=$((git_config_count + 1))
|
||||
done
|
||||
GIT_ENV+=("GIT_CONFIG_COUNT=$git_config_count")
|
||||
|
||||
(
|
||||
cd "$TMP_DIR"
|
||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||
"${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||
mapfile -t GIT_PACKAGES < <(
|
||||
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
|
||||
const fs = require("fs");
|
||||
@@ -108,7 +136,7 @@ NODE
|
||||
)
|
||||
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
|
||||
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
|
||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
||||
"${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
||||
fi
|
||||
)
|
||||
|
||||
|
||||
112
tools/release/generate-release-sbom.py
Normal file
112
tools/release/generate-release-sbom.py
Normal file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a CycloneDX SBOM from the resolved GovOPlaN release environment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
|
||||
META_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||
|
||||
from govoplan_release.sbom import ( # noqa: E402
|
||||
build_release_sbom,
|
||||
parse_sbom_timestamp,
|
||||
render_sbom,
|
||||
sbom_sha256,
|
||||
timestamp_from_source_date_epoch,
|
||||
validate_sbom,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--python", default=sys.executable, help="Python executable for pip inspect.")
|
||||
parser.add_argument("--core-root", type=Path, default=META_ROOT.parent / "govoplan-core")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=META_ROOT / "runtime" / "release-artifacts" / "govoplan-sbom.cdx.json",
|
||||
)
|
||||
parser.add_argument("--pip-inspect", type=Path, help="Use an existing pip inspect report instead of executing pip.")
|
||||
parser.add_argument(
|
||||
"--timestamp",
|
||||
help="ISO-8601 build timestamp. Defaults to SOURCE_DATE_EPOCH, then the current time.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
core_root = args.core_root.expanduser().resolve()
|
||||
product_version = _project_version(core_root / "pyproject.toml")
|
||||
npm_lock_path = core_root / "webui" / "package-lock.release.json"
|
||||
if not npm_lock_path.exists():
|
||||
parser.error(f"release npm lockfile does not exist: {npm_lock_path}")
|
||||
|
||||
pip_inspect = _read_json(args.pip_inspect) if args.pip_inspect else _run_pip_inspect(args.python)
|
||||
npm_lock = _read_json(npm_lock_path)
|
||||
try:
|
||||
timestamp = _resolved_timestamp(args.timestamp)
|
||||
payload = build_release_sbom(
|
||||
product_version=product_version,
|
||||
pip_inspect=pip_inspect,
|
||||
npm_lock=npm_lock,
|
||||
timestamp=timestamp,
|
||||
)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
validate_sbom(payload)
|
||||
|
||||
output = args.output.expanduser().resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(render_sbom(payload), encoding="utf-8")
|
||||
print(f"Wrote {len(payload['components'])} components to {output}")
|
||||
print(f"SHA-256: {sbom_sha256(payload)}")
|
||||
return 0
|
||||
|
||||
|
||||
def _run_pip_inspect(python: str) -> dict:
|
||||
result = subprocess.run(
|
||||
[python, "-m", "pip", "inspect", "--local"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"pip inspect failed: {result.stderr.strip()}")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"expected a JSON object in {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _project_version(path: Path) -> str:
|
||||
with path.open("rb") as handle:
|
||||
payload = tomllib.load(handle)
|
||||
project = payload.get("project")
|
||||
if not isinstance(project, dict) or not isinstance(project.get("version"), str):
|
||||
raise ValueError(f"project.version is missing from {path}")
|
||||
return project["version"]
|
||||
|
||||
|
||||
def _resolved_timestamp(explicit: str | None):
|
||||
if explicit is not None:
|
||||
return parse_sbom_timestamp(explicit)
|
||||
source_date_epoch = os.environ.get("SOURCE_DATE_EPOCH")
|
||||
if source_date_epoch is not None:
|
||||
return timestamp_from_source_date_epoch(source_date_epoch)
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -99,6 +99,7 @@ def collect_versions(path: Path) -> VersionSnapshot:
|
||||
package=read_json_version(path / "package.json"),
|
||||
webui_package=read_json_version(path / "webui" / "package.json"),
|
||||
manifests=read_manifest_versions(path),
|
||||
package_inits=read_package_init_versions(path),
|
||||
)
|
||||
|
||||
|
||||
@@ -137,6 +138,21 @@ def read_manifest_versions(path: Path) -> tuple[str, ...]:
|
||||
return tuple(versions)
|
||||
|
||||
|
||||
def read_package_init_versions(path: Path) -> tuple[str, ...]:
|
||||
"""Read public runtime versions declared by top-level Python packages."""
|
||||
|
||||
src = path / "src"
|
||||
if not src.exists():
|
||||
return ()
|
||||
versions: list[str] = []
|
||||
for package_init in sorted(src.glob("*/__init__.py")):
|
||||
text = package_init.read_text(encoding="utf-8")
|
||||
match = re.search(r'(?m)^__version__\s*=\s*["\']([^"\']+)["\']', text)
|
||||
if match is not None:
|
||||
versions.append(match.group(1))
|
||||
return tuple(versions)
|
||||
|
||||
|
||||
def git_text(path: Path, *args: str, timeout: int = 10) -> str:
|
||||
result = git(path, *args, timeout=timeout)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
@@ -26,10 +26,17 @@ class VersionSnapshot:
|
||||
package: str | None = None
|
||||
webui_package: str | None = None
|
||||
manifests: tuple[str, ...] = ()
|
||||
package_inits: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def primary(self) -> str | None:
|
||||
return self.pyproject or self.package or self.webui_package or (self.manifests[0] if self.manifests else None)
|
||||
return (
|
||||
self.pyproject
|
||||
or self.package
|
||||
or self.webui_package
|
||||
or (self.manifests[0] if self.manifests else None)
|
||||
or (self.package_inits[0] if self.package_inits else None)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
||||
@@ -13,7 +13,9 @@ from govoplan_core.core.module_package_catalog import validate_module_package_ca
|
||||
|
||||
from .catalog import canonical_hash
|
||||
from .model import CatalogPublishResult, CatalogPublishStep
|
||||
from .module_directory import write_module_directory
|
||||
from .selective_catalog import trusted_keys_from_keyring
|
||||
from .version_alignment import candidate_catalog_version_issues
|
||||
from .workspace import DEFAULT_WORKSPACE_ROOT, resolve_workspace_root, website_root
|
||||
|
||||
|
||||
@@ -41,7 +43,6 @@ def publish_catalog_candidate(
|
||||
resolved_web_root = Path(web_root).expanduser().resolve() if web_root is not None else website_root(workspace)
|
||||
candidate_catalog = candidate_root / "channels" / f"{channel}.json"
|
||||
candidate_keyring = candidate_root / "keyring.json"
|
||||
candidate_modules = candidate_root / "modules"
|
||||
target_catalog = resolved_web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json"
|
||||
target_keyring = resolved_web_root / "public" / "catalogs" / "v1" / "keyring.json"
|
||||
target_modules = resolved_web_root / "public" / "catalogs" / "v1" / "modules"
|
||||
@@ -72,11 +73,36 @@ def publish_catalog_candidate(
|
||||
validation_warnings: tuple[str, ...] = ()
|
||||
|
||||
if candidate_payload is not None and keyring_payload is not None:
|
||||
version_issues = candidate_catalog_version_issues(candidate_payload)
|
||||
blockers.extend(
|
||||
f"candidate version alignment: {issue.source}: {issue.actual!r}; "
|
||||
f"expected {issue.expected!r} ({issue.message})"
|
||||
for issue in version_issues
|
||||
)
|
||||
candidate_keys = trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {})
|
||||
target_keyring_payload = read_json(target_keyring) if target_keyring.exists() else None
|
||||
publication_trust = trusted_keys_from_keyring(
|
||||
target_keyring_payload if isinstance(target_keyring_payload, dict) else {}
|
||||
)
|
||||
if not publication_trust:
|
||||
blockers.append(
|
||||
"publication trust anchor is missing; bootstrap a trusted website keyring through a separate reviewed process"
|
||||
)
|
||||
for key_id in sorted(set(candidate_keys) & set(publication_trust)):
|
||||
if candidate_keys[key_id] != publication_trust[key_id]:
|
||||
blockers.append(f"candidate keyring changes the public key for trusted key id {key_id!r}")
|
||||
if candidate_keyring_hash != target_keyring_hash_before:
|
||||
release = candidate_payload.get("release") if isinstance(candidate_payload, dict) else None
|
||||
signed_keyring_hash = release.get("keyring_sha256") if isinstance(release, dict) else None
|
||||
if signed_keyring_hash != candidate_keyring_hash:
|
||||
blockers.append(
|
||||
"candidate keyring differs from the publication trust anchor without a matching signed keyring_sha256"
|
||||
)
|
||||
validation = validate_module_package_catalog(
|
||||
candidate_catalog,
|
||||
require_trusted=True,
|
||||
approved_channels=(channel,),
|
||||
trusted_keys=trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {}),
|
||||
trusted_keys=publication_trust,
|
||||
)
|
||||
validation_valid = validation.get("valid") is True
|
||||
validation_error = str(validation["error"]) if validation.get("error") else None
|
||||
@@ -115,14 +141,12 @@ def publish_catalog_candidate(
|
||||
steps.append(
|
||||
CatalogPublishStep(
|
||||
id="copy",
|
||||
title="Copy candidate catalog, keyring, and module directory",
|
||||
title="Copy candidate catalog/keyring and derive the module directory",
|
||||
detail=f"{candidate_root} -> {resolved_web_root / 'public' / 'catalogs' / 'v1'}",
|
||||
mutating=True,
|
||||
status="planned" if not apply else "blocked" if blockers else "done",
|
||||
)
|
||||
)
|
||||
if not candidate_modules.exists():
|
||||
notes.append("Candidate has no module-directory tree; only catalog and keyring will be copied.")
|
||||
if build_web:
|
||||
steps.append(
|
||||
CatalogPublishStep(
|
||||
@@ -224,10 +248,15 @@ def publish_catalog_candidate(
|
||||
target_catalog.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(candidate_catalog, target_catalog)
|
||||
shutil.copy2(candidate_keyring, target_keyring)
|
||||
if candidate_modules.exists():
|
||||
if target_modules.exists():
|
||||
shutil.rmtree(target_modules)
|
||||
shutil.copytree(candidate_modules, target_modules)
|
||||
write_module_directory(
|
||||
catalog_payload=candidate_payload,
|
||||
keyring_payload=keyring_payload,
|
||||
output_root=target_catalog.parent.parent,
|
||||
channel=channel,
|
||||
public_base_url=catalog_public_base_url(candidate_payload),
|
||||
)
|
||||
|
||||
completed_steps = [replace_status(step, "done") if step.id == "copy" else step for step in steps]
|
||||
if build_web:
|
||||
@@ -282,6 +311,14 @@ def publish_catalog_candidate(
|
||||
)
|
||||
|
||||
|
||||
def catalog_public_base_url(payload: dict[str, object]) -> str:
|
||||
release = payload.get("release")
|
||||
catalog_url = release.get("catalog_url") if isinstance(release, dict) else None
|
||||
if isinstance(catalog_url, str) and "/catalogs/" in catalog_url:
|
||||
return catalog_url.split("/catalogs/", 1)[0]
|
||||
return "https://govoplan.add-ideas.de"
|
||||
|
||||
|
||||
def result(
|
||||
*,
|
||||
status: str,
|
||||
|
||||
235
tools/release/govoplan_release/sbom.py
Normal file
235
tools/release/govoplan_release/sbom.py
Normal file
@@ -0,0 +1,235 @@
|
||||
"""CycloneDX SBOM construction from resolved Python and npm environments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlsplit
|
||||
import uuid
|
||||
|
||||
|
||||
def build_release_sbom(
|
||||
*,
|
||||
product_version: str,
|
||||
pip_inspect: dict[str, Any],
|
||||
npm_lock: dict[str, Any],
|
||||
timestamp: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
components = _deduplicate_components(
|
||||
[
|
||||
*python_components(pip_inspect),
|
||||
*npm_components(npm_lock),
|
||||
]
|
||||
)
|
||||
root_ref = f"pkg:generic/govoplan@{quote(product_version, safe='') }"
|
||||
resolved_timestamp = _normalized_timestamp(timestamp)
|
||||
timestamp_text = resolved_timestamp.isoformat().replace("+00:00", "Z")
|
||||
serial_seed = json.dumps(
|
||||
{
|
||||
"product_version": product_version,
|
||||
"timestamp": timestamp_text,
|
||||
"components": components,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
serial = uuid.uuid5(uuid.NAMESPACE_URL, f"https://govoplan.add-ideas.de/sbom/{serial_seed}")
|
||||
return {
|
||||
"$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json",
|
||||
"bomFormat": "CycloneDX",
|
||||
"specVersion": "1.6",
|
||||
"serialNumber": f"urn:uuid:{serial}",
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"timestamp": timestamp_text,
|
||||
"lifecycles": [{"phase": "build"}],
|
||||
"component": {
|
||||
"type": "application",
|
||||
"bom-ref": root_ref,
|
||||
"group": "ADD ideas UG",
|
||||
"name": "GovOPlaN",
|
||||
"version": product_version,
|
||||
"purl": root_ref,
|
||||
},
|
||||
},
|
||||
"components": components,
|
||||
"dependencies": [
|
||||
{
|
||||
"ref": root_ref,
|
||||
"dependsOn": [component["bom-ref"] for component in components],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def python_components(report: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if str(report.get("version")) != "1":
|
||||
raise ValueError("unsupported pip inspect report version")
|
||||
installed = report.get("installed")
|
||||
if not isinstance(installed, list):
|
||||
raise ValueError("pip inspect report is missing installed distributions")
|
||||
|
||||
components: list[dict[str, Any]] = []
|
||||
for item in installed:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
metadata = item.get("metadata")
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
name = metadata.get("name")
|
||||
version = metadata.get("version")
|
||||
if not isinstance(name, str) or not isinstance(version, str):
|
||||
continue
|
||||
normalized = name.lower().replace("_", "-")
|
||||
purl = f"pkg:pypi/{quote(normalized, safe='')}@{quote(version, safe='')}"
|
||||
component: dict[str, Any] = {
|
||||
"type": "library",
|
||||
"bom-ref": purl,
|
||||
"name": name,
|
||||
"version": version,
|
||||
"purl": purl,
|
||||
"scope": "required",
|
||||
}
|
||||
direct_url = item.get("direct_url")
|
||||
if isinstance(direct_url, dict) and isinstance(direct_url.get("url"), str):
|
||||
distribution_url = _public_distribution_url(direct_url["url"])
|
||||
if distribution_url is not None:
|
||||
component["externalReferences"] = [{"type": "distribution", "url": distribution_url}]
|
||||
components.append(component)
|
||||
return components
|
||||
|
||||
|
||||
def npm_components(lock: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
packages = lock.get("packages")
|
||||
if not isinstance(packages, dict):
|
||||
raise ValueError("npm lockfile is missing the packages map")
|
||||
|
||||
components: list[dict[str, Any]] = []
|
||||
for path, item in packages.items():
|
||||
if not path or not isinstance(item, dict):
|
||||
continue
|
||||
name = item.get("name") or _npm_name_from_lock_path(str(path))
|
||||
version = item.get("version")
|
||||
if not isinstance(name, str) or not isinstance(version, str):
|
||||
continue
|
||||
purl = f"pkg:npm/{quote(name, safe='/')}@{quote(version, safe='')}"
|
||||
component: dict[str, Any] = {
|
||||
"type": "library",
|
||||
"bom-ref": purl,
|
||||
"name": name,
|
||||
"version": version,
|
||||
"purl": purl,
|
||||
"scope": "optional" if item.get("dev") is True else "required",
|
||||
}
|
||||
hashes = _sri_hashes(item.get("integrity"))
|
||||
if hashes:
|
||||
component["hashes"] = hashes
|
||||
resolved = item.get("resolved")
|
||||
if isinstance(resolved, str):
|
||||
distribution_url = _public_distribution_url(resolved)
|
||||
if distribution_url is not None:
|
||||
component["externalReferences"] = [{"type": "distribution", "url": distribution_url}]
|
||||
components.append(component)
|
||||
return components
|
||||
|
||||
|
||||
def validate_sbom(payload: dict[str, Any]) -> None:
|
||||
if payload.get("bomFormat") != "CycloneDX" or payload.get("specVersion") != "1.6":
|
||||
raise ValueError("not a supported CycloneDX 1.6 SBOM")
|
||||
metadata = payload.get("metadata")
|
||||
if not isinstance(metadata, dict) or not isinstance(metadata.get("component"), dict):
|
||||
raise ValueError("SBOM metadata component is missing")
|
||||
components = payload.get("components")
|
||||
if not isinstance(components, list) or not components:
|
||||
raise ValueError("SBOM has no dependency components")
|
||||
refs = [item.get("bom-ref") for item in components if isinstance(item, dict)]
|
||||
if any(not isinstance(ref, str) or not ref for ref in refs):
|
||||
raise ValueError("SBOM component is missing bom-ref")
|
||||
if len(refs) != len(set(refs)):
|
||||
raise ValueError("SBOM contains duplicate component references")
|
||||
|
||||
|
||||
def sbom_sha256(payload: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(render_sbom(payload).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def render_sbom(payload: dict[str, Any]) -> str:
|
||||
return json.dumps(payload, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
def timestamp_from_source_date_epoch(value: str) -> datetime:
|
||||
"""Resolve the reproducible-build timestamp convention to UTC."""
|
||||
|
||||
try:
|
||||
epoch = int(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError("SOURCE_DATE_EPOCH must be an integer Unix timestamp") from exc
|
||||
try:
|
||||
return datetime.fromtimestamp(epoch, tz=UTC)
|
||||
except (OverflowError, OSError) as exc:
|
||||
raise ValueError("SOURCE_DATE_EPOCH is outside the supported timestamp range") from exc
|
||||
|
||||
|
||||
def parse_sbom_timestamp(value: str) -> datetime:
|
||||
"""Parse an explicit ISO-8601 SBOM timestamp and normalize it to UTC."""
|
||||
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError("timestamp must be an ISO-8601 value with a timezone") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("timestamp must include a timezone")
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _normalized_timestamp(value: datetime | None) -> datetime:
|
||||
resolved = value or datetime.now(tz=UTC)
|
||||
if resolved.tzinfo is None:
|
||||
raise ValueError("SBOM timestamp must include a timezone")
|
||||
return resolved.astimezone(UTC).replace(microsecond=0)
|
||||
|
||||
|
||||
def _deduplicate_components(components: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
unique = {component["bom-ref"]: component for component in components}
|
||||
return [unique[key] for key in sorted(unique)]
|
||||
|
||||
|
||||
def _npm_name_from_lock_path(path: str) -> str:
|
||||
marker = "node_modules/"
|
||||
if marker not in path:
|
||||
return Path(path).name
|
||||
return path.rsplit(marker, 1)[-1]
|
||||
|
||||
|
||||
def _sri_hashes(value: object) -> list[dict[str, str]]:
|
||||
if not isinstance(value, str) or "-" not in value:
|
||||
return []
|
||||
algorithm, encoded = value.split("-", 1)
|
||||
algorithm_name = {"sha256": "SHA-256", "sha384": "SHA-384", "sha512": "SHA-512"}.get(algorithm.lower())
|
||||
if algorithm_name is None:
|
||||
return []
|
||||
try:
|
||||
content = base64.b64decode(encoded, validate=True).hex()
|
||||
except ValueError:
|
||||
return []
|
||||
return [{"alg": algorithm_name, "content": content}]
|
||||
|
||||
|
||||
def _public_distribution_url(value: str) -> str | None:
|
||||
"""Keep stable public provenance without leaking local paths or credentials."""
|
||||
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.scheme.lower() not in {"git", "git+https", "http", "https"}:
|
||||
return None
|
||||
if not parsed.hostname or parsed.username is not None or parsed.password is not None:
|
||||
return None
|
||||
if parsed.query:
|
||||
return None
|
||||
return value
|
||||
@@ -19,6 +19,7 @@ from .contracts import collect_contracts
|
||||
from .git_state import collect_repository_snapshot
|
||||
from .model import CatalogEntryChange, SelectiveCatalogCandidate
|
||||
from .module_directory import write_module_directory
|
||||
from .version_alignment import selected_repository_version_issues
|
||||
from .workspace import load_repository_specs, resolve_workspace_root, website_root
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||
@@ -46,6 +47,7 @@ def build_selective_catalog_candidate(
|
||||
raise ValueError("At least one signing key is required.")
|
||||
|
||||
workspace = resolve_workspace_root(workspace_root)
|
||||
enforce_selected_version_alignment(repo_versions=repo_versions, workspace=workspace)
|
||||
web_root = website_root(workspace)
|
||||
source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel)
|
||||
payload = read_catalog(source_catalog)
|
||||
@@ -75,22 +77,24 @@ def build_selective_catalog_candidate(
|
||||
repository_base=repository_base.rstrip("/"),
|
||||
)
|
||||
|
||||
candidate.pop("signature", None)
|
||||
candidate.pop("signatures", None)
|
||||
candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys]
|
||||
|
||||
output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at)
|
||||
catalog_path = output_root / "channels" / f"{channel}.json"
|
||||
keyring_path = output_root / "keyring.json"
|
||||
summary_path = output_root / "summary.json" if write_summary else None
|
||||
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
keyring_payload = keyring_payload_for_candidate(
|
||||
existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json",
|
||||
signing_keys=parsed_keys,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
release["keyring_sha256"] = canonical_hash(keyring_payload)
|
||||
|
||||
candidate.pop("signature", None)
|
||||
candidate.pop("signatures", None)
|
||||
candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys]
|
||||
|
||||
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
module_directory_files = write_module_directory(
|
||||
catalog_payload=candidate,
|
||||
@@ -159,6 +163,18 @@ def build_selective_catalog_candidate(
|
||||
return result
|
||||
|
||||
|
||||
def enforce_selected_version_alignment(*, repo_versions: dict[str, str], workspace: Path) -> None:
|
||||
failures = [
|
||||
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
|
||||
for issue in selected_repository_version_issues(
|
||||
repo_versions=repo_versions,
|
||||
workspace=workspace,
|
||||
)
|
||||
]
|
||||
if failures:
|
||||
raise ValueError("Version alignment gate failed: " + "; ".join(failures))
|
||||
|
||||
|
||||
def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str:
|
||||
if base_catalog is not None:
|
||||
value = str(base_catalog)
|
||||
|
||||
@@ -76,6 +76,19 @@ def build_unit(repo: RepositorySnapshot, *, target_version: str | None, contract
|
||||
blockers.append(f"repository is behind {repo.upstream}")
|
||||
if repo.exists and repo.is_git and not repo.has_head:
|
||||
blockers.append("repository has no initial commit")
|
||||
local_versions = tuple(
|
||||
value
|
||||
for value in (
|
||||
repo.versions.pyproject,
|
||||
repo.versions.package,
|
||||
repo.versions.webui_package,
|
||||
*repo.versions.manifests,
|
||||
*repo.versions.package_inits,
|
||||
)
|
||||
if value
|
||||
)
|
||||
if len(set(local_versions)) > 1:
|
||||
blockers.append("backend, manifest, and frontend version metadata is not aligned")
|
||||
if repo.dirty:
|
||||
warnings.append("uncommitted changes will need review before release")
|
||||
if repo.ahead:
|
||||
|
||||
652
tools/release/govoplan_release/version_alignment.py
Normal file
652
tools/release/govoplan_release/version_alignment.py
Normal file
@@ -0,0 +1,652 @@
|
||||
"""Version-alignment checks for independently released GovOPlaN packages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tomllib
|
||||
|
||||
from .git_state import collect_versions
|
||||
from .workspace import load_repository_specs, resolve_repo_path
|
||||
|
||||
|
||||
_PYTHON_RELEASE_REF = re.compile(
|
||||
r"^(?P<package>govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://.+/"
|
||||
r"(?P<repo>govoplan-[a-z0-9-]+)\.git@v(?P<version>[^\s;]+)$"
|
||||
)
|
||||
_WEBUI_RELEASE_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git#v(?P<version>[^#\s]+)$")
|
||||
_CATALOG_PYTHON_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git@v(?P<version>[^\s;]+)$")
|
||||
_CATALOG_WEBUI_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git#v(?P<version>[^#\s]+)$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class VersionAlignmentIssue:
|
||||
repo: str
|
||||
source: str
|
||||
expected: str
|
||||
actual: str
|
||||
message: str
|
||||
|
||||
|
||||
def repository_version_issues(
|
||||
repo_path: Path,
|
||||
*,
|
||||
expected_version: str | None = None,
|
||||
include_lockfiles: bool = True,
|
||||
) -> tuple[VersionAlignmentIssue, ...]:
|
||||
"""Compare every version-bearing file in one source repository."""
|
||||
|
||||
versions = collect_versions(repo_path)
|
||||
declared = {
|
||||
"pyproject.toml": versions.pyproject,
|
||||
"package.json": versions.package,
|
||||
"webui/package.json": versions.webui_package,
|
||||
"webui/package.release.json": _json_version(repo_path / "webui" / "package.release.json")
|
||||
if (repo_path / "webui" / "package.release.json").exists()
|
||||
else None,
|
||||
}
|
||||
for index, version in enumerate(versions.manifests, start=1):
|
||||
declared[f"module manifest {index}"] = version
|
||||
for index, version in enumerate(versions.package_inits, start=1):
|
||||
declared[f"package __init__.py {index}"] = version
|
||||
|
||||
canonical_source, canonical_version = next(
|
||||
((source, version) for source, version in declared.items() if version),
|
||||
(None, None),
|
||||
)
|
||||
repo = repo_path.name
|
||||
if canonical_source is None or canonical_version is None:
|
||||
if expected_version is None:
|
||||
return ()
|
||||
return (
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source="repository",
|
||||
expected=expected_version.removeprefix("v"),
|
||||
actual="",
|
||||
message="repository has no version metadata",
|
||||
),
|
||||
)
|
||||
|
||||
issues = [
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=source,
|
||||
expected=canonical_version,
|
||||
actual=version,
|
||||
message=f"{source} must match {canonical_source}",
|
||||
)
|
||||
for source, version in declared.items()
|
||||
if version is not None and version != canonical_version
|
||||
]
|
||||
|
||||
if expected_version is not None and canonical_version.removeprefix("v") != expected_version.removeprefix("v"):
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=canonical_source,
|
||||
expected=expected_version.removeprefix("v"),
|
||||
actual=canonical_version,
|
||||
message="source version must match the requested release version",
|
||||
)
|
||||
)
|
||||
|
||||
if include_lockfiles:
|
||||
issues.extend(
|
||||
_lockfile_issues(
|
||||
repo=repo,
|
||||
package_path=repo_path / "package.json",
|
||||
lock_path=repo_path / "package-lock.json",
|
||||
)
|
||||
)
|
||||
issues.extend(
|
||||
_lockfile_issues(
|
||||
repo=repo,
|
||||
package_path=repo_path / "webui" / "package.json",
|
||||
lock_path=repo_path / "webui" / "package-lock.json",
|
||||
)
|
||||
)
|
||||
issues.extend(
|
||||
_lockfile_issues(
|
||||
repo=repo,
|
||||
package_path=repo_path / "webui" / "package.release.json",
|
||||
lock_path=repo_path / "webui" / "package-lock.release.json",
|
||||
)
|
||||
)
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def selected_repository_version_issues(
|
||||
*,
|
||||
repo_versions: dict[str, str],
|
||||
workspace: Path,
|
||||
include_lockfiles: bool = True,
|
||||
) -> tuple[VersionAlignmentIssue, ...]:
|
||||
"""Validate registered selected worktrees against explicit release versions."""
|
||||
|
||||
specs = {spec.name: spec for spec in load_repository_specs(include_website=False)}
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
for repo, expected_version in sorted(repo_versions.items()):
|
||||
spec = specs.get(repo)
|
||||
if spec is None:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source="repository",
|
||||
expected=expected_version.removeprefix("v"),
|
||||
actual="",
|
||||
message="repository is not registered",
|
||||
)
|
||||
)
|
||||
continue
|
||||
repo_path = resolve_repo_path(spec, workspace)
|
||||
if not repo_path.exists():
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source="repository",
|
||||
expected=expected_version.removeprefix("v"),
|
||||
actual="",
|
||||
message="repository path is missing",
|
||||
)
|
||||
)
|
||||
continue
|
||||
issues.extend(
|
||||
repository_version_issues(
|
||||
repo_path,
|
||||
expected_version=expected_version,
|
||||
include_lockfiles=include_lockfiles,
|
||||
)
|
||||
)
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def release_composition_issues(meta_root: Path, *, core_root: Path) -> tuple[VersionAlignmentIssue, ...]:
|
||||
"""Require release references to agree and point at installable artifacts."""
|
||||
|
||||
python_refs = _python_release_refs(meta_root / "requirements-release.txt")
|
||||
webui_refs = _webui_release_refs(core_root / "webui" / "package.release.json")
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
|
||||
for repo in sorted(set(python_refs) & set(webui_refs)):
|
||||
python_version = python_refs[repo]
|
||||
webui_version = webui_refs[repo]
|
||||
if python_version == webui_version:
|
||||
continue
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source="webui/package.release.json",
|
||||
expected=python_version,
|
||||
actual=webui_version,
|
||||
message="frontend release ref must match the backend release ref",
|
||||
)
|
||||
)
|
||||
workspace = meta_root.parent
|
||||
for repo, version in sorted(python_refs.items()):
|
||||
repo_path = workspace / repo
|
||||
if repo_path.exists():
|
||||
issues.extend(
|
||||
_release_artifact_issues(
|
||||
repo=repo,
|
||||
repo_path=repo_path,
|
||||
version=version,
|
||||
artifact="pyproject.toml",
|
||||
)
|
||||
)
|
||||
for repo, version in sorted(webui_refs.items()):
|
||||
repo_path = workspace / repo
|
||||
if repo_path.exists():
|
||||
issues.extend(
|
||||
_release_artifact_issues(
|
||||
repo=repo,
|
||||
repo_path=repo_path,
|
||||
version=version,
|
||||
artifact="webui/package.json",
|
||||
)
|
||||
)
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def _release_artifact_issues(
|
||||
*,
|
||||
repo: str,
|
||||
repo_path: Path,
|
||||
version: str,
|
||||
artifact: str,
|
||||
) -> list[VersionAlignmentIssue]:
|
||||
tag = f"v{version.removeprefix('v')}"
|
||||
payload = _git_tag_file(repo_path, tag, artifact)
|
||||
if payload is None:
|
||||
return [
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{tag}:{artifact}",
|
||||
expected="installable package metadata",
|
||||
actual="missing or unreadable",
|
||||
message="release tag must contain its declared package artifact",
|
||||
)
|
||||
]
|
||||
try:
|
||||
if artifact == "pyproject.toml":
|
||||
parsed = tomllib.loads(payload)
|
||||
project = parsed.get("project")
|
||||
artifact_version = project.get("version") if isinstance(project, dict) else None
|
||||
else:
|
||||
parsed = json.loads(payload)
|
||||
artifact_version = parsed.get("version") if isinstance(parsed, dict) else None
|
||||
except (json.JSONDecodeError, tomllib.TOMLDecodeError):
|
||||
artifact_version = None
|
||||
if artifact_version == version.removeprefix("v"):
|
||||
return []
|
||||
return [
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{tag}:{artifact}",
|
||||
expected=version.removeprefix("v"),
|
||||
actual=str(artifact_version or ""),
|
||||
message="release artifact version must match its immutable tag",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def candidate_catalog_version_issues(payload: object) -> tuple[VersionAlignmentIssue, ...]:
|
||||
"""Validate versions, refs, and selected-unit provenance inside a catalog."""
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return (
|
||||
VersionAlignmentIssue(
|
||||
repo="catalog",
|
||||
source="catalog",
|
||||
expected="object",
|
||||
actual=type(payload).__name__,
|
||||
message="candidate catalog must be a JSON object",
|
||||
),
|
||||
)
|
||||
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
represented: dict[str, str] = {}
|
||||
core_release = payload.get("core_release")
|
||||
if isinstance(core_release, dict):
|
||||
issues.extend(_catalog_entry_issues(core_release, source="core_release", represented=represented))
|
||||
else:
|
||||
issues.append(_catalog_shape_issue("core_release", "candidate catalog has no core release"))
|
||||
|
||||
modules = payload.get("modules")
|
||||
if not isinstance(modules, list):
|
||||
issues.append(_catalog_shape_issue("modules", "candidate catalog has no modules list"))
|
||||
else:
|
||||
for index, entry in enumerate(modules):
|
||||
if not isinstance(entry, dict):
|
||||
issues.append(_catalog_shape_issue(f"modules[{index}]", "module entry must be an object"))
|
||||
continue
|
||||
issues.extend(
|
||||
_catalog_entry_issues(
|
||||
entry,
|
||||
source=f"modules[{index}]",
|
||||
represented=represented,
|
||||
)
|
||||
)
|
||||
|
||||
release = payload.get("release")
|
||||
if isinstance(release, dict):
|
||||
issues.extend(_catalog_release_issues(release, represented=represented))
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def _catalog_entry_issues(
|
||||
entry: dict[str, object],
|
||||
*,
|
||||
source: str,
|
||||
represented: dict[str, str],
|
||||
) -> list[VersionAlignmentIssue]:
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
version = entry.get("version")
|
||||
normalized_version = version.removeprefix("v") if isinstance(version, str) and version else None
|
||||
if normalized_version is None:
|
||||
issues.append(_catalog_shape_issue(f"{source}.version", "catalog entry has no version"))
|
||||
|
||||
python_ref = entry.get("python_ref")
|
||||
python_match = _CATALOG_PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None
|
||||
if python_match is None:
|
||||
issues.append(_catalog_shape_issue(f"{source}.python_ref", "Python ref must end in a version tag"))
|
||||
repo = str(entry.get("python_package") or entry.get("module_id") or source)
|
||||
else:
|
||||
repo = python_match.group("repo")
|
||||
ref_version = python_match.group("version").removeprefix("v")
|
||||
if normalized_version is not None and ref_version != normalized_version:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{source}.python_ref",
|
||||
expected=normalized_version,
|
||||
actual=ref_version,
|
||||
message="Python ref tag must match the catalog entry version",
|
||||
)
|
||||
)
|
||||
if normalized_version is not None:
|
||||
previous = represented.setdefault(repo, normalized_version)
|
||||
if previous != normalized_version:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=source,
|
||||
expected=previous,
|
||||
actual=normalized_version,
|
||||
message="repository appears with conflicting catalog versions",
|
||||
)
|
||||
)
|
||||
|
||||
webui_package = entry.get("webui_package")
|
||||
webui_ref = entry.get("webui_ref")
|
||||
if bool(webui_package) != bool(webui_ref):
|
||||
issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI package and ref must be declared together"))
|
||||
if isinstance(webui_ref, str):
|
||||
webui_match = _CATALOG_WEBUI_REF.search(webui_ref)
|
||||
if webui_match is None:
|
||||
issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI ref must end in a version tag"))
|
||||
else:
|
||||
webui_repo = webui_match.group("repo")
|
||||
webui_version = webui_match.group("version").removeprefix("v")
|
||||
if python_match is not None and webui_repo != repo:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{source}.webui_ref",
|
||||
expected=repo,
|
||||
actual=webui_repo,
|
||||
message="backend and WebUI refs must use the same repository",
|
||||
)
|
||||
)
|
||||
if normalized_version is not None and webui_version != normalized_version:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{source}.webui_ref",
|
||||
expected=normalized_version,
|
||||
actual=webui_version,
|
||||
message="WebUI ref tag must match the catalog entry version",
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def _catalog_release_issues(
|
||||
release: dict[str, object],
|
||||
*,
|
||||
represented: dict[str, str],
|
||||
) -> list[VersionAlignmentIssue]:
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
version = release.get("version")
|
||||
tag = release.get("tag")
|
||||
if isinstance(version, str) and isinstance(tag, str) and tag != f"v{version.removeprefix('v')}":
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo="govoplan-core",
|
||||
source="release.tag",
|
||||
expected=f"v{version.removeprefix('v')}",
|
||||
actual=tag,
|
||||
message="release tag must match release version",
|
||||
)
|
||||
)
|
||||
|
||||
selected_units = release.get("selected_units")
|
||||
if selected_units is None:
|
||||
return issues
|
||||
if not isinstance(selected_units, list):
|
||||
issues.append(_catalog_shape_issue("release.selected_units", "selected units must be a list"))
|
||||
return issues
|
||||
seen: set[str] = set()
|
||||
for index, unit in enumerate(selected_units):
|
||||
if not isinstance(unit, dict):
|
||||
issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit must be an object"))
|
||||
continue
|
||||
repo = unit.get("repo")
|
||||
unit_version = unit.get("version")
|
||||
unit_tag = unit.get("tag")
|
||||
if not isinstance(repo, str) or not isinstance(unit_version, str) or not isinstance(unit_tag, str):
|
||||
issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit requires repo, version, and tag"))
|
||||
continue
|
||||
normalized_version = unit_version.removeprefix("v")
|
||||
if repo in seen:
|
||||
issues.append(_catalog_shape_issue(f"release.selected_units[{index}].repo", "selected repository is duplicated"))
|
||||
seen.add(repo)
|
||||
if unit_tag != f"v{normalized_version}":
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"release.selected_units[{index}].tag",
|
||||
expected=f"v{normalized_version}",
|
||||
actual=unit_tag,
|
||||
message="selected-unit tag must match its version",
|
||||
)
|
||||
)
|
||||
represented_version = represented.get(repo)
|
||||
if represented_version is None:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"release.selected_units[{index}]",
|
||||
expected=normalized_version,
|
||||
actual="",
|
||||
message="selected repository is not represented in the catalog",
|
||||
)
|
||||
)
|
||||
elif represented_version != normalized_version:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"release.selected_units[{index}].version",
|
||||
expected=represented_version,
|
||||
actual=normalized_version,
|
||||
message="selected-unit version must match the catalog entry",
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def _catalog_shape_issue(source: str, message: str) -> VersionAlignmentIssue:
|
||||
return VersionAlignmentIssue(
|
||||
repo="catalog",
|
||||
source=source,
|
||||
expected="valid value",
|
||||
actual="",
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def _lockfile_issues(*, repo: str, package_path: Path, lock_path: Path) -> list[VersionAlignmentIssue]:
|
||||
if not package_path.exists() or not lock_path.exists():
|
||||
return []
|
||||
package_payload = _json_object(package_path)
|
||||
lock_payload = _json_object(lock_path)
|
||||
package_version = _payload_version(package_payload)
|
||||
lock_version = _lockfile_root_version(lock_path)
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
if package_version is None or lock_version is None or package_version == lock_version:
|
||||
pass
|
||||
else:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=str(lock_path.relative_to(package_path.parents[1] if package_path.parent.name == "webui" else package_path.parent)),
|
||||
expected=package_version,
|
||||
actual=lock_version,
|
||||
message=f"lockfile root version must match {package_path.name}",
|
||||
)
|
||||
)
|
||||
lock_packages = lock_payload.get("packages")
|
||||
lock_root = lock_packages.get("") if isinstance(lock_packages, dict) else None
|
||||
if isinstance(lock_root, dict):
|
||||
for group in ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies"):
|
||||
package_dependencies = package_payload.get(group) or {}
|
||||
lock_dependencies = lock_root.get(group) or {}
|
||||
if package_dependencies != lock_dependencies:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{lock_path.name}:{group}",
|
||||
expected=json.dumps(package_dependencies, sort_keys=True),
|
||||
actual=json.dumps(lock_dependencies, sort_keys=True),
|
||||
message=f"lockfile root {group} must match {package_path.name}",
|
||||
)
|
||||
)
|
||||
issues.extend(
|
||||
_git_lock_resolution_issues(
|
||||
repo=repo,
|
||||
package_payload=package_payload,
|
||||
lock_packages=lock_packages,
|
||||
workspace=package_path.parents[2] if package_path.parent.name == "webui" else package_path.parent.parent,
|
||||
lock_name=lock_path.name,
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def _git_lock_resolution_issues(
|
||||
*,
|
||||
repo: str,
|
||||
package_payload: dict[str, object],
|
||||
lock_packages: dict[str, object],
|
||||
workspace: Path,
|
||||
lock_name: str,
|
||||
) -> list[VersionAlignmentIssue]:
|
||||
dependencies = package_payload.get("dependencies")
|
||||
if not isinstance(dependencies, dict):
|
||||
return []
|
||||
issues: list[VersionAlignmentIssue] = []
|
||||
for package_name, spec in dependencies.items():
|
||||
if not isinstance(package_name, str) or not isinstance(spec, str):
|
||||
continue
|
||||
match = _CATALOG_WEBUI_REF.search(spec)
|
||||
if match is None:
|
||||
continue
|
||||
source_repo = match.group("repo")
|
||||
version = match.group("version").removeprefix("v")
|
||||
locked = lock_packages.get(f"node_modules/{package_name}")
|
||||
if not isinstance(locked, dict):
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{lock_name}:{package_name}",
|
||||
expected=version,
|
||||
actual="",
|
||||
message="release lock has no resolved GovOPlaN package",
|
||||
)
|
||||
)
|
||||
continue
|
||||
locked_version = locked.get("version")
|
||||
if locked_version != version:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{lock_name}:{package_name}",
|
||||
expected=version,
|
||||
actual=str(locked_version or ""),
|
||||
message="resolved package version must match its release ref",
|
||||
)
|
||||
)
|
||||
expected_commit = _git_tag_commit(workspace / source_repo, f"v{version}")
|
||||
resolved = locked.get("resolved")
|
||||
resolved_commit = resolved.rsplit("#", 1)[-1] if isinstance(resolved, str) and "#" in resolved else None
|
||||
if expected_commit is None or resolved_commit != expected_commit:
|
||||
issues.append(
|
||||
VersionAlignmentIssue(
|
||||
repo=repo,
|
||||
source=f"{lock_name}:{package_name}:resolved",
|
||||
expected=expected_commit or f"local tag v{version}",
|
||||
actual=resolved_commit or "",
|
||||
message="resolved package commit must match the local release tag",
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def _git_tag_commit(repo_path: Path, tag: str) -> str | None:
|
||||
if not (repo_path / ".git").exists():
|
||||
return None
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_path), "rev-list", "-n", "1", tag],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
)
|
||||
value = result.stdout.strip()
|
||||
return value if result.returncode == 0 and value else None
|
||||
|
||||
|
||||
def _git_tag_file(repo_path: Path, tag: str, relative_path: str) -> str | None:
|
||||
if not (repo_path / ".git").exists():
|
||||
return None
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_path), "show", f"{tag}:{relative_path}"],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
)
|
||||
return result.stdout if result.returncode == 0 else None
|
||||
|
||||
|
||||
def _json_version(path: Path) -> str | None:
|
||||
value = _payload_version(_json_object(path))
|
||||
return value
|
||||
|
||||
|
||||
def _payload_version(payload: dict[str, object]) -> str | None:
|
||||
value = payload.get("version")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _json_object(path: Path) -> dict[str, object]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _lockfile_root_version(path: Path) -> str | None:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
packages = payload.get("packages")
|
||||
if isinstance(packages, dict):
|
||||
root = packages.get("")
|
||||
if isinstance(root, dict) and isinstance(root.get("version"), str):
|
||||
return root["version"]
|
||||
value = payload.get("version")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _python_release_refs(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
refs: dict[str, str] = {}
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
match = _PYTHON_RELEASE_REF.match(line.strip())
|
||||
if match is None:
|
||||
continue
|
||||
repo = match.group("repo")
|
||||
refs[repo] = match.group("version")
|
||||
return refs
|
||||
|
||||
|
||||
def _webui_release_refs(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
dependencies = payload.get("dependencies") if isinstance(payload, dict) else None
|
||||
if not isinstance(dependencies, dict):
|
||||
return {}
|
||||
refs: dict[str, str] = {}
|
||||
for spec in dependencies.values():
|
||||
if not isinstance(spec, str):
|
||||
continue
|
||||
match = _WEBUI_RELEASE_REF.search(spec)
|
||||
if match is not None:
|
||||
refs[match.group("repo")] = match.group("version")
|
||||
return refs
|
||||
@@ -194,6 +194,7 @@ run() {
|
||||
}
|
||||
|
||||
GEN_ARGS=(
|
||||
env "GOVOPLAN_CORE_ROOT=$CORE_ROOT"
|
||||
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
|
||||
--version "$VERSION"
|
||||
--channel "$CHANNEL"
|
||||
|
||||
@@ -52,28 +52,10 @@ Options:
|
||||
-h, --help Show this help.
|
||||
|
||||
Repos:
|
||||
Installable release packages:
|
||||
govoplan-access
|
||||
govoplan-admin
|
||||
govoplan-tenancy
|
||||
govoplan-organizations
|
||||
govoplan-identity
|
||||
govoplan-idm
|
||||
govoplan-policy
|
||||
govoplan-audit
|
||||
govoplan-dashboard
|
||||
govoplan-files
|
||||
govoplan-mail
|
||||
govoplan-campaign
|
||||
govoplan-calendar
|
||||
govoplan-ops
|
||||
govoplan-core
|
||||
|
||||
Roadmap/scaffold module repositories are tag-only until they have package
|
||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
xrechnung, and xta-osci.
|
||||
Core, every registered release/module repository listed below in this script,
|
||||
and the meta repository. Repositories with pyproject.toml are versioned and
|
||||
alignment-gated dynamically; remaining roadmap/scaffold repositories are
|
||||
committed and tagged without inventing package metadata.
|
||||
|
||||
Examples:
|
||||
tools/release/push-release-tag.sh
|
||||
@@ -121,12 +103,14 @@ PACKAGE_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-organizations"
|
||||
"$PARENT/govoplan-permits"
|
||||
"$PARENT/govoplan-policy"
|
||||
"$PARENT/govoplan-poll"
|
||||
"$PARENT/govoplan-procurement"
|
||||
"$PARENT/govoplan-records"
|
||||
"$PARENT/govoplan-resources"
|
||||
"$PARENT/govoplan-risk-compliance"
|
||||
"$PARENT/govoplan-tenancy"
|
||||
"$PARENT/govoplan-transparency"
|
||||
"$PARENT/govoplan-evaluation"
|
||||
)
|
||||
TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-addresses"
|
||||
@@ -134,6 +118,7 @@ TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-cases"
|
||||
"$PARENT/govoplan-connectors"
|
||||
"$PARENT/govoplan-dms"
|
||||
"$PARENT/govoplan-dist-lists"
|
||||
"$PARENT/govoplan-erp"
|
||||
"$PARENT/govoplan-fit-connect"
|
||||
"$PARENT/govoplan-forms"
|
||||
@@ -144,8 +129,10 @@ TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-portal"
|
||||
"$PARENT/govoplan-postbox"
|
||||
"$PARENT/govoplan-reporting"
|
||||
"$PARENT/govoplan-rest"
|
||||
"$PARENT/govoplan-scheduling"
|
||||
"$PARENT/govoplan-search"
|
||||
"$PARENT/govoplan-soap"
|
||||
"$PARENT/govoplan-tasks"
|
||||
"$PARENT/govoplan-templates"
|
||||
"$PARENT/govoplan-workflow"
|
||||
@@ -154,10 +141,12 @@ TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-xta-osci"
|
||||
)
|
||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
||||
SUPPORT_REPOS=("$META_ROOT")
|
||||
PACKAGE_REPOS=()
|
||||
REPOS=(
|
||||
"${MODULE_REPOS[@]}"
|
||||
"$ROOT"
|
||||
"${SUPPORT_REPOS[@]}"
|
||||
)
|
||||
|
||||
REMOTE="origin"
|
||||
@@ -379,6 +368,7 @@ if project_name != "govoplan-core":
|
||||
peers["@govoplan/core-webui"] = f"^{new_version}"
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
synchronize_lockfile_root "$package_path" "${package_path%package.json}package-lock.json"
|
||||
done
|
||||
|
||||
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
||||
@@ -401,9 +391,59 @@ for name, spec in list(dependencies.items()):
|
||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
synchronize_lockfile_root "$release_package_path" "$repo/webui/package-lock.release.json"
|
||||
fi
|
||||
}
|
||||
|
||||
synchronize_lockfile_root() {
|
||||
local package_path="$1"
|
||||
local lock_path="$2"
|
||||
[[ -f "$package_path" && -f "$lock_path" ]] || return 0
|
||||
"$PYTHON" - "$package_path" "$lock_path" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
package_path = Path(sys.argv[1])
|
||||
lock_path = Path(sys.argv[2])
|
||||
package = json.loads(package_path.read_text())
|
||||
lock = json.loads(lock_path.read_text())
|
||||
version = package.get("version")
|
||||
if not isinstance(version, str) or not version:
|
||||
raise SystemExit(f"package version is missing from {package_path}")
|
||||
lock["version"] = version
|
||||
packages = lock.get("packages")
|
||||
if isinstance(packages, dict) and isinstance(packages.get(""), dict):
|
||||
packages[""]["version"] = version
|
||||
lock_path.write_text(json.dumps(lock, indent=2) + "\n")
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_release_requirements() {
|
||||
local version="$1"
|
||||
"$PYTHON" - "$META_ROOT/requirements-release.txt" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
version = sys.argv[2]
|
||||
text = path.read_text()
|
||||
updated, count = re.subn(
|
||||
r"(?m)^(govoplan-[a-z0-9-]+\s+@\s+git\+ssh://[^\s]+\.git)@v[^\s]+$",
|
||||
rf"\1@v{version}",
|
||||
text,
|
||||
)
|
||||
if count == 0:
|
||||
raise SystemExit(f"no tagged GovOPlaN requirements found in {path}")
|
||||
path.write_text(updated)
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_version_files() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
@@ -427,7 +467,30 @@ generate_release_lock() {
|
||||
fi
|
||||
|
||||
[[ -x "$META_ROOT/tools/release/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $META_ROOT/tools/release/generate-release-lock.sh"
|
||||
run "$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN"
|
||||
local command=("$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN")
|
||||
local repo
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
command+=(--local-git-repo "$repo")
|
||||
done
|
||||
run "${command[@]}"
|
||||
}
|
||||
|
||||
run_version_alignment_gate() {
|
||||
local mode="${1:-full}"
|
||||
local command=(
|
||||
"$PYTHON"
|
||||
"$META_ROOT/tools/checks/check-version-alignment.py"
|
||||
--workspace-root "$PARENT"
|
||||
--release-composition
|
||||
)
|
||||
if [[ "$mode" == "source" ]]; then
|
||||
command+=(--source-metadata-only)
|
||||
fi
|
||||
local repo
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
command+=(--repo-version "$(basename "$repo")=$TARGET_VERSION")
|
||||
done
|
||||
run "${command[@]}"
|
||||
}
|
||||
|
||||
run_migration_release_audit() {
|
||||
@@ -694,6 +757,15 @@ for repo in "${REPOS[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
EFFECTIVE_TAG_ONLY_REPOS=()
|
||||
for repo in "${REPOS[@]}"; do
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
PACKAGE_REPOS+=("$repo")
|
||||
else
|
||||
EFFECTIVE_TAG_ONLY_REPOS+=("$repo")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$TARGET_VERSION" ]]; then
|
||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
@@ -744,9 +816,9 @@ echo " tag: $TAG"
|
||||
echo " remote: $REMOTE"
|
||||
echo " commit message: $COMMIT_MESSAGE"
|
||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
||||
echo " tag-only/support repos: ${#EFFECTIVE_TAG_ONLY_REPOS[@]} committed/tagged/pushed without package version files"
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
||||
echo " release lock: resolve core webui/package-lock.release.json from local reviewed module tags"
|
||||
else
|
||||
echo " release lock: skipped"
|
||||
fi
|
||||
@@ -780,9 +852,21 @@ for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
|
||||
else
|
||||
update_release_requirements "$TARGET_VERSION"
|
||||
fi
|
||||
|
||||
refresh_development_webui_lock
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
# Source metadata, release refs, and lock roots must agree before creating any
|
||||
# release commit or tag. A second gate below verifies the fully resolved release
|
||||
# lock before any remote mutation.
|
||||
run_version_alignment_gate source
|
||||
|
||||
PRE_CORE_REPOS=("${MODULE_REPOS[@]}" "${SUPPORT_REPOS[@]}")
|
||||
for repo in "${PRE_CORE_REPOS[@]}"; do
|
||||
run git -C "$repo" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
@@ -806,12 +890,13 @@ for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
done
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||
done
|
||||
|
||||
generate_release_lock
|
||||
|
||||
# npm resolves the new module tags from the local repositories. Nothing has
|
||||
# been pushed yet; a failed lock or alignment check leaves only local,
|
||||
# recoverable commits/tags.
|
||||
run_version_alignment_gate
|
||||
|
||||
run git -C "$ROOT" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
@@ -822,6 +907,10 @@ fi
|
||||
|
||||
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
|
||||
for repo in "${PRE_CORE_REPOS[@]}"; do
|
||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||
done
|
||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
|
||||
@@ -381,7 +381,7 @@ def repo_findings(repos: tuple[RepoState, ...], *, target_version: str | None, t
|
||||
findings.append(
|
||||
Finding(
|
||||
check_id="repo-version-mismatch",
|
||||
severity="warning",
|
||||
severity="blocker",
|
||||
title=f"{repo.name} has inconsistent local version files",
|
||||
detail=", ".join(
|
||||
value
|
||||
|
||||
Reference in New Issue
Block a user