Compare commits
11 Commits
17f8036bdc
...
99e255cf81
| Author | SHA1 | Date | |
|---|---|---|---|
| 99e255cf81 | |||
| d9819c4aad | |||
| 78b601a2e5 | |||
| a18c7ea040 | |||
| b193d4555f | |||
| a81dc21e90 | |||
| 676030b993 | |||
| a15a74c54c | |||
| c34892f6ea | |||
| 397c87a012 | |||
| f1b07354a0 |
16
.env.example
16
.env.example
@@ -15,7 +15,23 @@ REDIS_URL=redis://127.0.0.1:6379/0
|
|||||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||||
|
|
||||||
|
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=false
|
||||||
|
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||||
|
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||||
|
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||||
|
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||||
|
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||||
|
GOVOPLAN_HTTP_HSTS_SECONDS=31536000
|
||||||
|
|
||||||
|
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||||
|
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||||
|
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||||
|
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||||
|
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||||
|
|
||||||
CORS_ORIGINS=https://govoplan.example.org
|
CORS_ORIGINS=https://govoplan.example.org
|
||||||
|
GOVOPLAN_TRUSTED_HOSTS=govoplan.example.org
|
||||||
|
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||||
AUTH_COOKIE_SECURE=true
|
AUTH_COOKIE_SECURE=true
|
||||||
AUTH_COOKIE_SAMESITE=lax
|
AUTH_COOKIE_SAMESITE=lax
|
||||||
AUTH_COOKIE_DOMAIN=
|
AUTH_COOKIE_DOMAIN=
|
||||||
|
|||||||
24
README.md
24
README.md
@@ -78,6 +78,27 @@ Run the static cross-repository module contract check:
|
|||||||
./tools/checks/check-contracts.sh
|
./tools/checks/check-contracts.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Require backend, manifest, frontend, lockfile, and release-composition versions
|
||||||
|
to agree before a release:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./.venv/bin/python tools/checks/check-version-alignment.py --release-composition
|
||||||
|
```
|
||||||
|
|
||||||
|
Generate the CycloneDX dependency inventory from a resolved release environment:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
||||||
|
```
|
||||||
|
|
||||||
|
For reproducible release artifacts, set `SOURCE_DATE_EPOCH` to the release
|
||||||
|
commit timestamp (or pass an explicit timezone-qualified `--timestamp`):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
SOURCE_DATE_EPOCH="$(git -C ../govoplan-core show -s --format=%ct HEAD)" \
|
||||||
|
./.venv/bin/python tools/release/generate-release-sbom.py --python ./.venv/bin/python
|
||||||
|
```
|
||||||
|
|
||||||
Run the consolidated focused verification suite:
|
Run the consolidated focused verification suite:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -128,6 +149,9 @@ Frontend layout principles for module pages are documented in
|
|||||||
The cross-product destination, stakeholder visions, configuration archetypes,
|
The cross-product destination, stakeholder visions, configuration archetypes,
|
||||||
connected outcome stories, and capability horizons are documented in
|
connected outcome stories, and capability horizons are documented in
|
||||||
the [Connected Governance Platform Roadmap](docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md).
|
the [Connected Governance Platform Roadmap](docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md).
|
||||||
|
The administrator journey from Core-only bootstrap through online module
|
||||||
|
installation, scale-out, and reversible environment promotion is defined in
|
||||||
|
[System Administrator Lifecycle User Story](docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md).
|
||||||
The first Campaign-centric capability and infrastructure fit assessment is in
|
The first Campaign-centric capability and infrastructure fit assessment is in
|
||||||
`docs/CAPABILITY_AND_INFRASTRUCTURE_FIT.md`.
|
`docs/CAPABILITY_AND_INFRASTRUCTURE_FIT.md`.
|
||||||
|
|
||||||
|
|||||||
@@ -19,8 +19,24 @@ CELERY_ENABLED=true
|
|||||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||||
|
|
||||||
|
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=true
|
||||||
|
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||||
|
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||||
|
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||||
|
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||||
|
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||||
|
GOVOPLAN_HTTP_HSTS_SECONDS=0
|
||||||
|
|
||||||
|
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||||
|
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||||
|
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||||
|
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||||
|
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||||
|
|
||||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||||
|
GOVOPLAN_TRUSTED_HOSTS=127.0.0.1,localhost,testserver
|
||||||
|
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||||
AUTH_COOKIE_SECURE=false
|
AUTH_COOKIE_SECURE=false
|
||||||
FILE_STORAGE_BACKEND=local
|
FILE_STORAGE_BACKEND=local
|
||||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||||
|
|||||||
@@ -253,6 +253,12 @@ Operational access is distinct from permission to read all business content.
|
|||||||
Diagnostics minimize personal data and link to governed evidence when deeper
|
Diagnostics minimize personal data and link to governed evidence when deeper
|
||||||
inspection is authorized.
|
inspection is authorized.
|
||||||
|
|
||||||
|
The complete installation and lifecycle journey is specified in the
|
||||||
|
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
|
||||||
|
one-command Core-only bootstrap, signed online module installation and updates,
|
||||||
|
stateless scale-out, versioned configuration transfer, undo, and reproducible
|
||||||
|
environment-promotion recipes.
|
||||||
|
|
||||||
### Integration owner and enterprise architect
|
### Integration owner and enterprise architect
|
||||||
|
|
||||||
> As an integration owner, I can inventory external systems, declare direction
|
> As an integration owner, I can inventory external systems, declare direction
|
||||||
@@ -871,23 +877,27 @@ checkouts.
|
|||||||
|
|
||||||
Priorities:
|
Priorities:
|
||||||
|
|
||||||
1. Pin and publish a compatible Core/WebUI/module composition and first
|
1. Deliver the first slices of the
|
||||||
|
[System Administrator Lifecycle User Story](SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md):
|
||||||
|
a verified Core-only distribution, first-run control plane, read-only online
|
||||||
|
module directory, and durable plan/confirm/install progress.
|
||||||
|
2. Pin and publish a compatible Core/WebUI/module composition and first
|
||||||
reference configuration package.
|
reference configuration package.
|
||||||
2. Complete database, files, configuration, secrets, and audit backup/restore
|
3. Complete database, files, configuration, secrets, and audit backup/restore
|
||||||
drills with measured recovery objectives.
|
drills with measured recovery objectives.
|
||||||
3. Prove the providers required by the selected reference configurations,
|
4. Prove the providers required by the selected reference configurations,
|
||||||
beginning with SMTP/IMAP, CalDAV, and the relevant broker/worker and file or
|
beginning with SMTP/IMAP, CalDAV, and the relevant broker/worker and file or
|
||||||
directory adapters, including reconciliation behavior.
|
directory adapters, including reconciliation behavior.
|
||||||
4. Add external monitoring, alerting, audit export/retention, deployment
|
5. Add external monitoring, alerting, audit export/retention, deployment
|
||||||
hardening, and incident/runbook evidence.
|
hardening, and incident/runbook evidence.
|
||||||
5. Apply the interface pattern language first to Campaign, scheduling, and
|
6. Apply the interface pattern language first to Campaign, scheduling, and
|
||||||
admin/configuration. Existing central components are mandatory; any custom
|
admin/configuration. Existing central components are mandatory; any custom
|
||||||
control needs prior product-owner authorization and a narrow recorded scope.
|
control needs prior product-owner authorization and a narrow recorded scope.
|
||||||
Extend central contracts only after a repeated need is demonstrated.
|
Extend central contracts only after a repeated need is demonstrated.
|
||||||
6. Align executable WebUI contributions with manifest/configured-system
|
7. Align executable WebUI contributions with manifest/configured-system
|
||||||
metadata and make focused views manually/default selectable without waiting
|
metadata and make focused views manually/default selectable without waiting
|
||||||
for Workflow.
|
for Workflow.
|
||||||
7. Refresh the capability/infrastructure assessment and distinguish code-tested,
|
8. Refresh the capability/infrastructure assessment and distinguish code-tested,
|
||||||
target-tested, package-integrated, pilot-approved, and production-approved
|
target-tested, package-integrated, pilot-approved, and production-approved
|
||||||
evidence.
|
evidence.
|
||||||
|
|
||||||
|
|||||||
@@ -151,6 +151,24 @@ forcing every repository to the same tag, but channel publication must preserve
|
|||||||
the unchanged package versions, validate interface compatibility, sign the
|
the unchanged package versions, validate interface compatibility, sign the
|
||||||
updated catalog, and keep the published keyring healthy.
|
updated catalog, and keep the published keyring healthy.
|
||||||
|
|
||||||
|
Release integration also enforces repository and composition version alignment
|
||||||
|
and generates a CycloneDX SBOM from the resolved Python environment and the
|
||||||
|
release WebUI lockfile. Catalog publication should attach that immutable SBOM
|
||||||
|
and its digest to the corresponding Core/composition release.
|
||||||
|
|
||||||
|
Addresses and Notifications have current WebUI source contributions but are
|
||||||
|
not part of the pinned v0.1.8 WebUI release composition because their v0.1.8
|
||||||
|
tags predate those packages. They re-enter the release composition only through
|
||||||
|
new, immutable module tags whose backend, manifest, frontend, and lock metadata
|
||||||
|
pass the alignment gate.
|
||||||
|
|
||||||
|
Candidate publication verifies signatures against the already published
|
||||||
|
keyring, not against keys supplied only by the candidate. A changed keyring
|
||||||
|
must have its canonical SHA-256 embedded in the signed catalog. The public
|
||||||
|
module-directory files are regenerated from that verified catalog and keyring
|
||||||
|
at publication time; candidate-supplied directory files are never copied as
|
||||||
|
authoritative provenance.
|
||||||
|
|
||||||
## Published Module Directory
|
## Published Module Directory
|
||||||
|
|
||||||
The target release repository is an online, browsable module directory. The
|
The target release repository is an online, browsable module directory. The
|
||||||
|
|||||||
@@ -34,16 +34,28 @@ tools/checks/security-audit/run.sh --mode full --scope govoplan
|
|||||||
```
|
```
|
||||||
|
|
||||||
Reports are written to `audit-reports/`, which is intentionally ignored by git.
|
Reports are written to `audit-reports/`, which is intentionally ignored by git.
|
||||||
|
Each run records tool versions, report checksums, and start/end repository
|
||||||
|
revision plus worktree fingerprints. A repository change during scanning makes
|
||||||
|
the run fail so a mixed code snapshot cannot be reported as a valid audit.
|
||||||
|
Step exit codes are recorded separately from findings: report-only mode may
|
||||||
|
accept findings, but scanner execution errors and malformed JSON/SARIF reports
|
||||||
|
always fail the run. The manifest lists the expected, present, and missing
|
||||||
|
reports for that invocation. Validation and checksums use that explicit set, so
|
||||||
|
reusing a report directory cannot make stale output look like part of a new run.
|
||||||
|
|
||||||
The wrapper tags the toolbox image by a fingerprint of the Dockerfile and
|
The wrapper tags the toolbox image by a fingerprint of the Dockerfile,
|
||||||
`requirements-audit.txt`. If those inputs have not changed, subsequent runs reuse
|
`requirements-audit.txt`, and the Semgrep smoke-test inputs. If those inputs have
|
||||||
the existing local image instead of reinstalling all tools. The stable alias is
|
not changed, subsequent runs reuse the existing local image instead of
|
||||||
`govoplan/security-audit:local` unless `SECURITY_AUDIT_IMAGE` is set.
|
reinstalling all tools. The stable alias is `govoplan/security-audit:local`
|
||||||
|
unless `SECURITY_AUDIT_IMAGE` is set.
|
||||||
|
|
||||||
Semgrep is installed separately in the toolbox image because current Semgrep
|
Semgrep is installed separately in the toolbox image because current Semgrep
|
||||||
packages pin Click to an affected `8.1.x` line. The image upgrades Click after
|
packages pin affected Click and MCP versions. The image upgrades both after
|
||||||
Semgrep installation and fails during build if Semgrep no longer starts with the
|
Semgrep installation, runs an actual local-rule scan, and audits the final
|
||||||
patched Click version.
|
toolbox Python environment during the image build. The compatibility releases
|
||||||
|
are pinned exactly to keep the tested override reproducible. Remove either
|
||||||
|
compatibility override only after Semgrep's own dependency bounds include a
|
||||||
|
fixed version.
|
||||||
|
|
||||||
Force a cached rebuild:
|
Force a cached rebuild:
|
||||||
|
|
||||||
@@ -70,6 +82,10 @@ tools/checks/security-audit/run.sh --mode quick --scope current --update --build
|
|||||||
- `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit.
|
- `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit.
|
||||||
- `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon.
|
- `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon.
|
||||||
|
|
||||||
|
Semgrep and Trivy are invoked with finding-sensitive exit codes. Their exit 1
|
||||||
|
is therefore a finding under the wrapper contract; higher exit codes, missing
|
||||||
|
output, invalid JSON/SARIF, and scanner error payloads are execution failures.
|
||||||
|
|
||||||
Bandit and Ruff security reports are split by source kind. Production code under
|
Bandit and Ruff security reports are split by source kind. Production code under
|
||||||
`src/` is written to `bandit.json` and `ruff-security.json` and controls strict
|
`src/` is written to `bandit.json` and `ruff-security.json` and controls strict
|
||||||
mode. Test code under `tests/` is still scanned for visibility, but its findings
|
mode. Test code under `tests/` is still scanned for visibility, but its findings
|
||||||
@@ -153,7 +169,10 @@ cd /mnt/DATA/git/govoplan
|
|||||||
python -m venv .venv
|
python -m venv .venv
|
||||||
./.venv/bin/python -m pip install -r requirements-audit.txt
|
./.venv/bin/python -m pip install -r requirements-audit.txt
|
||||||
./.venv/bin/python -m pip install 'semgrep>=1.140,<2'
|
./.venv/bin/python -m pip install 'semgrep>=1.140,<2'
|
||||||
./.venv/bin/python -m pip install --upgrade --no-deps 'click>=8.3.3'
|
./.venv/bin/python -m pip install --upgrade --no-deps 'click==8.3.3'
|
||||||
|
./.venv/bin/python -m pip install --upgrade --no-deps 'mcp==1.28.1'
|
||||||
|
./.venv/bin/semgrep scan --metrics=off --config tools/checks/security-audit/semgrep-govoplan.yml tools/checks/check-version-alignment.py
|
||||||
|
./.venv/bin/pip-audit --progress-spinner off
|
||||||
```
|
```
|
||||||
|
|
||||||
Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`)
|
Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`)
|
||||||
|
|||||||
169
docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md
Normal file
169
docs/SYSTEM_ADMINISTRATOR_LIFECYCLE_USER_STORY.md
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
# System Administrator Lifecycle User Story
|
||||||
|
|
||||||
|
## Story
|
||||||
|
|
||||||
|
> As a system administrator, I can execute one shell command that downloads a
|
||||||
|
> verified GovOPlaN distribution and starts a completely configured Core control
|
||||||
|
> plane without optional modules. In the WebUI I can browse compatible signed
|
||||||
|
> module releases, select the modules for this installation, and follow every
|
||||||
|
> download, validation, migration, installation, activation, and health-check
|
||||||
|
> step. When an update is available, I can review its impact and confirm it.
|
||||||
|
>
|
||||||
|
> I can add API or worker replicas on containers or other servers without
|
||||||
|
> copying mutable local state. I can operate development, test, and production
|
||||||
|
> systems, transfer a versioned configuration through the WebUI, undo an applied
|
||||||
|
> configuration version, test an update in one environment, and then apply the
|
||||||
|
> same immutable update recipe to another environment.
|
||||||
|
|
||||||
|
This is a product-level story owned by the GovOPlaN platform rather than by an
|
||||||
|
individual domain module. It joins installation, module lifecycle, operations,
|
||||||
|
configuration packages, and release provenance into one administrator journey.
|
||||||
|
The canonical backlog item is
|
||||||
|
[GovOPlaN #13](https://git.add-ideas.de/add-ideas/govoplan/issues/13).
|
||||||
|
|
||||||
|
## Terms
|
||||||
|
|
||||||
|
- **Core control plane:** the smallest bootable distribution: Core API, Core
|
||||||
|
WebUI, PostgreSQL, Redis, installer worker, migration runner, and durable
|
||||||
|
storage configuration. No optional GovOPlaN module package is installed.
|
||||||
|
- **Bootstrap administrator:** a single-use, time-limited installation identity
|
||||||
|
that may access only first-run and module-lifecycle functions. It is retired
|
||||||
|
when the selected identity/access configuration becomes healthy.
|
||||||
|
- **Module release:** an immutable, signed package plus manifest, compatibility
|
||||||
|
contract, migrations, WebUI contribution, checksums, and SBOM references.
|
||||||
|
- **Configuration revision:** an immutable, schema-versioned export of
|
||||||
|
non-secret system/module settings, policies, compositions, and secret
|
||||||
|
references. Secret values are never exported.
|
||||||
|
- **Update recipe:** the reviewed, immutable plan containing exact Core/module
|
||||||
|
versions, configuration revision, migration order, preflight results,
|
||||||
|
maintenance/drain requirements, health checks, and permitted rollback or
|
||||||
|
forward-recovery actions.
|
||||||
|
|
||||||
|
## Acceptance journeys
|
||||||
|
|
||||||
|
### One-command first installation
|
||||||
|
|
||||||
|
1. The administrator runs one documented command on a supported host.
|
||||||
|
2. The bootstrapper verifies a signed distribution manifest before executing or
|
||||||
|
starting downloaded artifacts.
|
||||||
|
3. It checks container/runtime, ports, storage, entropy, memory, architecture,
|
||||||
|
and connectivity requirements; generates deployment-local secrets with
|
||||||
|
restrictive permissions; and never prints them.
|
||||||
|
4. It starts PostgreSQL, Redis, Core API/WebUI, one installer worker, and the
|
||||||
|
migration runner. Readiness does not pass until migrations and durable
|
||||||
|
dependencies are healthy.
|
||||||
|
5. It prints the local URL and one-time bootstrap credential. Re-running the
|
||||||
|
command is idempotent and shows or repairs the existing installation rather
|
||||||
|
than creating another identity or database.
|
||||||
|
6. No optional module is installed or enabled at this point.
|
||||||
|
|
||||||
|
### Module selection, installation, and update
|
||||||
|
|
||||||
|
1. Core reads an approved signed catalog and trusted keyring through the shared
|
||||||
|
outbound-network policy.
|
||||||
|
2. The WebUI shows available, installed, compatible, blocked, withdrawn, and
|
||||||
|
update-available releases with their channel, provenance, contracts,
|
||||||
|
migrations, permissions, configuration requirements, and release notes.
|
||||||
|
3. Selecting modules produces a dependency/compatibility plan before any
|
||||||
|
mutation. The administrator can amend the selection or confirm the plan.
|
||||||
|
4. Installation executes durably in a worker. The UI receives persisted step
|
||||||
|
state and can reconnect without losing progress.
|
||||||
|
5. Package signature/checksum, version alignment, contract closure, migration
|
||||||
|
graph, configuration schema, and health checks are mandatory gates.
|
||||||
|
6. A failed or interrupted operation reaches a visible retry, forward-recovery,
|
||||||
|
rollback, or manual-intervention state. It never reports success merely
|
||||||
|
because the initiating request returned.
|
||||||
|
7. An available update follows the same preview and confirmation path. Exact
|
||||||
|
repeated requests are idempotent.
|
||||||
|
|
||||||
|
### Horizontal scaling
|
||||||
|
|
||||||
|
1. API, WebUI, installer, scheduler, and ordinary worker roles are stateless
|
||||||
|
with respect to local container disks. Durable state uses PostgreSQL, Redis,
|
||||||
|
and configured shared file/object storage.
|
||||||
|
2. A documented command can add API or queue-specific worker replicas. The
|
||||||
|
default Compose profile supports local scale-out; an orchestrator profile
|
||||||
|
supplies equivalent health/readiness probes and rolling replacement.
|
||||||
|
3. Only one migration or module-lifecycle mutation may own the deployment lock,
|
||||||
|
while any healthy replica can serve read and normal domain traffic.
|
||||||
|
4. Workers announce identity, queues, software composition, heartbeat, and
|
||||||
|
drain state. Operators can see skew and safely retire a replica.
|
||||||
|
5. Sessions, throttling, idempotency, jobs, installer progress, and scheduled
|
||||||
|
work remain correct when requests move between replicas.
|
||||||
|
|
||||||
|
### Development, test, and production promotion
|
||||||
|
|
||||||
|
1. The administrator exports a configuration revision from development through
|
||||||
|
the WebUI. The package is versioned, checksummed, attributable, and contains
|
||||||
|
secret references or required-secret declarations, never secret values.
|
||||||
|
2. Test imports the package into a preview area. Core reports environment-bound
|
||||||
|
values, missing capabilities/secrets, compatibility changes, and the exact
|
||||||
|
apply plan.
|
||||||
|
3. Applying creates a new revision; the previous effective revision remains
|
||||||
|
addressable. Undo is a new audited revision that restores the earlier
|
||||||
|
configuration where contracts permit it.
|
||||||
|
4. After tests and health checks pass, Core emits an update recipe with exact
|
||||||
|
immutable release and configuration identifiers.
|
||||||
|
5. Production validates the recipe against its own environment, requires a new
|
||||||
|
confirmation, and executes the same ordered plan. Environment-specific
|
||||||
|
secret bindings and endpoints remain local.
|
||||||
|
6. Promotion never copies tenant/business data implicitly. Data migration,
|
||||||
|
anonymized fixtures, and backup/restore are separate explicit operations.
|
||||||
|
|
||||||
|
## Safety and governance requirements
|
||||||
|
|
||||||
|
- Catalogs, artifacts, recipes, and configuration revisions have signatures,
|
||||||
|
checksums, provenance, expiry/revocation semantics, and audit evidence.
|
||||||
|
- The installer has a dedicated narrow authority; normal domain permissions do
|
||||||
|
not imply host/package-management access.
|
||||||
|
- The UI distinguishes reversible configuration rollback from database or
|
||||||
|
package migrations that require forward recovery.
|
||||||
|
- Cluster-wide mutations use fencing/leases so an expired worker cannot later
|
||||||
|
commit a stale result.
|
||||||
|
- Update plans declare availability impact, worker draining, backup/restore
|
||||||
|
prerequisites, database compatibility windows, and post-change probes.
|
||||||
|
- Deployment profiles set secure headers, trusted proxies/hosts, body limits,
|
||||||
|
outbound-network policy, storage, TLS/cookie posture, and observability.
|
||||||
|
- Release publication requires aligned backend/frontend/manifest versions and
|
||||||
|
machine-readable dependency/SBOM provenance.
|
||||||
|
|
||||||
|
## Implementation slices
|
||||||
|
|
||||||
|
1. **Reproducible Core-only distribution.** Publish pinned multi-architecture
|
||||||
|
images, signed distribution manifest, Core-only Compose profile, bootstrap
|
||||||
|
preflight, generated secrets, readiness, and idempotent rerun/repair.
|
||||||
|
2. **First-run control plane.** Add the restricted bootstrap administrator,
|
||||||
|
one-time enrollment, initial catalog/keyring configuration, and retirement
|
||||||
|
after durable administrator access is established.
|
||||||
|
3. **Read-only online module directory.** Move the existing catalog and module
|
||||||
|
directory contracts into the installed Core WebUI with compatibility,
|
||||||
|
provenance, release-note, and update-state presentation.
|
||||||
|
4. **Durable module plan and install.** Reuse the existing installer queue,
|
||||||
|
locks, signed-package validator, rollback drill, and run evidence behind a
|
||||||
|
plan/confirm/progress UI. Add initial catalog-entry synthesis and artifact
|
||||||
|
acquisition where the current release console still assumes local sources.
|
||||||
|
5. **Safe module update.** Add drain/maintenance coordination, backup gate,
|
||||||
|
migration compatibility window, reconnectable progress, health verification,
|
||||||
|
retry/recovery, and update notification.
|
||||||
|
6. **Stateless replica profile.** Remove remaining local-runtime assumptions,
|
||||||
|
expose role-specific commands/images, implement worker registration/drain,
|
||||||
|
and prove multiple API and worker replicas against shared dependencies.
|
||||||
|
7. **Configuration revision model.** Define provider export/import schemas,
|
||||||
|
canonical serialization, secret references, validation/diff, immutable
|
||||||
|
revision storage, audit, apply, and undo-as-new-revision.
|
||||||
|
8. **Environment promotion and recipes.** Add source/target fingerprints,
|
||||||
|
preview, environment bindings, acceptance evidence, exact recipe generation,
|
||||||
|
signed transfer, and independently confirmed application.
|
||||||
|
9. **Operational proof.** Exercise interrupted installs, stale locks, unavailable
|
||||||
|
catalogs, revoked keys, failed migrations, replica loss, configuration undo,
|
||||||
|
and development-to-test-to-production promotion in release CI and target
|
||||||
|
drills.
|
||||||
|
|
||||||
|
## Explicit non-goals for the first distribution slice
|
||||||
|
|
||||||
|
- Shipping optional modules in the Core image.
|
||||||
|
- Exporting secrets or production business data with configuration.
|
||||||
|
- Pretending every schema migration can be reversed automatically.
|
||||||
|
- Building a proprietary orchestrator instead of supporting Compose and a
|
||||||
|
standard cluster scheduler through the same role/readiness contracts.
|
||||||
|
- Allowing the browser process to execute arbitrary shell commands.
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
# Whole-product release install from tagged module repositories.
|
# Whole-product release install from immutable, independently versioned module tags.
|
||||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when
|
# Only add a module after its referenced tag has been published.
|
||||||
# cutting a release.
|
|
||||||
../govoplan-core[server]
|
../govoplan-core[server]
|
||||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.8
|
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.8
|
||||||
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.8
|
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.8
|
||||||
@@ -11,14 +10,9 @@ govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.
|
|||||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.8
|
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.8
|
||||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.8
|
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.8
|
||||||
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.8
|
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.8
|
||||||
govoplan-addresses @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-addresses.git@v0.1.8
|
|
||||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.8
|
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.8
|
||||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.8
|
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.8
|
||||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.8
|
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.8
|
||||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.8
|
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.8
|
||||||
govoplan-poll @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-poll.git@v0.1.8
|
|
||||||
govoplan-scheduling @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-scheduling.git@v0.1.8
|
|
||||||
govoplan-notifications @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-notifications.git@v0.1.8
|
|
||||||
govoplan-evaluation @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-evaluation.git@v0.1.8
|
|
||||||
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.8
|
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.8
|
||||||
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.8
|
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.8
|
||||||
|
|||||||
99
tests/test_release_catalog_publication.py
Normal file
99
tests/test_release_catalog_publication.py
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||||
|
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||||
|
|
||||||
|
from govoplan_release.publisher import publish_catalog_candidate # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseCatalogPublicationTests(unittest.TestCase):
|
||||||
|
def test_publication_requires_an_existing_trust_anchor(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
candidate = self._candidate(root, key="candidate-key")
|
||||||
|
web_root = root / "website"
|
||||||
|
web_root.mkdir()
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_release.publisher.validate_module_package_catalog",
|
||||||
|
return_value={"valid": True, "warnings": [], "error": None},
|
||||||
|
):
|
||||||
|
result = publish_catalog_candidate(
|
||||||
|
candidate_dir=candidate,
|
||||||
|
web_root=web_root,
|
||||||
|
workspace_root=root,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.status)
|
||||||
|
self.assertIn("publication trust anchor is missing", " ".join(result.notes))
|
||||||
|
|
||||||
|
def test_publication_rejects_rebinding_an_existing_key_id(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
candidate = self._candidate(root, key="replacement-key")
|
||||||
|
web_root = root / "website"
|
||||||
|
target_keyring = web_root / "public" / "catalogs" / "v1" / "keyring.json"
|
||||||
|
target_keyring.parent.mkdir(parents=True)
|
||||||
|
target_keyring.write_text(json.dumps(self._keyring("trusted-key")))
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"govoplan_release.publisher.validate_module_package_catalog",
|
||||||
|
return_value={"valid": True, "warnings": [], "error": None},
|
||||||
|
):
|
||||||
|
result = publish_catalog_candidate(
|
||||||
|
candidate_dir=candidate,
|
||||||
|
web_root=web_root,
|
||||||
|
workspace_root=root,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", result.status)
|
||||||
|
self.assertIn("changes the public key", " ".join(result.notes))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _candidate(root: Path, *, key: str) -> Path:
|
||||||
|
candidate = root / "candidate"
|
||||||
|
channel = candidate / "channels"
|
||||||
|
channel.mkdir(parents=True)
|
||||||
|
channel.joinpath("stable.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"channel": "stable",
|
||||||
|
"core_release": {
|
||||||
|
"version": "1.2.3",
|
||||||
|
"python_ref": (
|
||||||
|
"govoplan-core @ git+ssh://git@example.test/acme/"
|
||||||
|
"govoplan-core.git@v1.2.3"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"modules": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
candidate.joinpath("keyring.json").write_text(json.dumps(ReleaseCatalogPublicationTests._keyring(key)))
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _keyring(key: str) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"keys": [
|
||||||
|
{
|
||||||
|
"key_id": "release-key",
|
||||||
|
"status": "active",
|
||||||
|
"public_key": key,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -51,6 +51,22 @@ class ReleaseDoctorTests(unittest.TestCase):
|
|||||||
self.assertEqual("repo-dirty", findings[0].check_id)
|
self.assertEqual("repo-dirty", findings[0].check_id)
|
||||||
self.assertIn("git status --short", [item.command for item in findings[0].commands])
|
self.assertIn("git status --short", [item.command for item in findings[0].commands])
|
||||||
|
|
||||||
|
def test_repo_version_mismatch_blocks_release(self) -> None:
|
||||||
|
doctor = load_doctor_module()
|
||||||
|
repo = doctor.RepoState(
|
||||||
|
name="govoplan-files",
|
||||||
|
path="/tmp/govoplan-files",
|
||||||
|
exists=True,
|
||||||
|
pyproject_version="0.1.9",
|
||||||
|
package_version="0.1.8",
|
||||||
|
webui_package_version="0.1.9",
|
||||||
|
)
|
||||||
|
|
||||||
|
findings = doctor.repo_findings((repo,), target_version="0.1.9", target_tag="v0.1.9")
|
||||||
|
|
||||||
|
mismatch = next(item for item in findings if item.check_id == "repo-version-mismatch")
|
||||||
|
self.assertEqual("blocker", mismatch.severity)
|
||||||
|
|
||||||
def test_dubious_ownership_errors_suggest_safe_directory_command(self) -> None:
|
def test_dubious_ownership_errors_suggest_safe_directory_command(self) -> None:
|
||||||
doctor = load_doctor_module()
|
doctor = load_doctor_module()
|
||||||
repo = doctor.RepoState(
|
repo = doctor.RepoState(
|
||||||
|
|||||||
67
tests/test_release_entrypoint_gates.py
Normal file
67
tests/test_release_entrypoint_gates.py
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseEntrypointGateTests(unittest.TestCase):
|
||||||
|
def test_lockstep_release_includes_every_registered_module_and_connector(self) -> None:
|
||||||
|
script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text()
|
||||||
|
registry = json.loads((META_ROOT / "repositories.json").read_text())
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
item["name"]
|
||||||
|
for item in registry["repositories"]
|
||||||
|
if item["category"] not in {"system", "website"}
|
||||||
|
}
|
||||||
|
missing = sorted(repo for repo in expected if f'$PARENT/{repo}"' not in script)
|
||||||
|
|
||||||
|
self.assertEqual([], missing)
|
||||||
|
|
||||||
|
def test_lockstep_release_runs_source_and_full_gates_before_remote_push(self) -> None:
|
||||||
|
script = (META_ROOT / "tools" / "release" / "push-release-tag.sh").read_text()
|
||||||
|
workflow = script.rsplit("\nconfirm_release\n", 1)[1]
|
||||||
|
|
||||||
|
source_gate = workflow.index("run_version_alignment_gate source")
|
||||||
|
first_commit = workflow.index('run git -C "$repo" commit')
|
||||||
|
lock_generation = workflow.index("generate_release_lock")
|
||||||
|
full_gate = workflow.index("run_version_alignment_gate", source_gate + 1)
|
||||||
|
first_push = workflow.index('run git -C "$repo" push')
|
||||||
|
|
||||||
|
self.assertLess(source_gate, first_commit)
|
||||||
|
self.assertLess(first_commit, lock_generation)
|
||||||
|
self.assertLess(lock_generation, full_gate)
|
||||||
|
self.assertLess(full_gate, first_push)
|
||||||
|
|
||||||
|
def test_source_catalog_generator_enforces_explicit_repo_versions(self) -> None:
|
||||||
|
script = (META_ROOT / "tools" / "release" / "generate-release-catalog.py").read_text()
|
||||||
|
|
||||||
|
gate = script.index("selected_repository_version_issues(")
|
||||||
|
write = script.index("output.write_text(")
|
||||||
|
|
||||||
|
self.assertLess(gate, write)
|
||||||
|
|
||||||
|
def test_candidate_publication_uses_existing_keyring_as_trust_anchor(self) -> None:
|
||||||
|
publisher = (META_ROOT / "tools" / "release" / "govoplan_release" / "publisher.py").read_text()
|
||||||
|
|
||||||
|
self.assertIn("candidate_catalog_version_issues(candidate_payload)", publisher)
|
||||||
|
self.assertIn("trusted_keys_from_keyring(\n target_keyring_payload", publisher)
|
||||||
|
self.assertIn("trusted_keys=publication_trust", publisher)
|
||||||
|
self.assertNotIn("trusted_keys=candidate_keys", publisher)
|
||||||
|
self.assertIn("write_module_directory(", publisher)
|
||||||
|
self.assertNotIn("shutil.copytree(candidate_modules", publisher)
|
||||||
|
|
||||||
|
def test_release_integration_honors_independent_module_versions(self) -> None:
|
||||||
|
script = (META_ROOT / "tools" / "checks" / "check-release-integration.sh").read_text()
|
||||||
|
|
||||||
|
self.assertNotIn('expected_tag = os.environ["RELEASE_TAG"]', script)
|
||||||
|
self.assertIn('dependency_version = tag_match.group(1)', script)
|
||||||
|
self.assertIn('repo_tag="$(release_tag_for_package "$repo")"', script)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
96
tests/test_release_integration.py
Normal file
96
tests/test_release_integration.py
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from tools.checks.release_integration import (
|
||||||
|
SOURCE_COUPLED_CORE_TESTS,
|
||||||
|
InstalledModuleArtifact,
|
||||||
|
artifact_contract_issues,
|
||||||
|
filter_test_suite,
|
||||||
|
release_package_names,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _named_test(test_id: str) -> unittest.TestCase:
|
||||||
|
class NamedTest(unittest.TestCase):
|
||||||
|
def id(self) -> str:
|
||||||
|
return test_id
|
||||||
|
|
||||||
|
def runTest(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return NamedTest()
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseIntegrationTests(unittest.TestCase):
|
||||||
|
def test_release_requirement_parser_selects_only_immutable_module_refs(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
requirements = Path(temp_dir) / "requirements-release.txt"
|
||||||
|
requirements.write_text(
|
||||||
|
"\n".join(
|
||||||
|
(
|
||||||
|
"../govoplan-core[server]",
|
||||||
|
"govoplan-idm @ git+ssh://git@example.test/add-ideas/govoplan-idm.git@v0.1.8",
|
||||||
|
"govoplan-campaign @ git+ssh://git@example.test/add-ideas/govoplan-campaign.git@v0.1.8",
|
||||||
|
"govoplan-idm @ git+ssh://git@example.test/add-ideas/govoplan-idm.git@v0.1.8",
|
||||||
|
"other-package==1.0",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
release_package_names(requirements),
|
||||||
|
("govoplan-idm", "govoplan-campaign"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_artifact_contracts_compare_distribution_entry_point_and_discovery_versions(self) -> None:
|
||||||
|
artifacts = (
|
||||||
|
InstalledModuleArtifact("govoplan-access", "0.1.8", "access", "0.1.8"),
|
||||||
|
InstalledModuleArtifact("govoplan-idm", "0.1.8", "idm", "0.1.7"),
|
||||||
|
InstalledModuleArtifact("govoplan-shadow-idm", "0.1.8", "idm", "0.1.8"),
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = artifact_contract_issues(
|
||||||
|
artifacts,
|
||||||
|
release_packages=("govoplan-access", "govoplan-idm", "govoplan-shadow-idm", "govoplan-campaign"),
|
||||||
|
available_manifest_versions={"access": "0.1.8", "idm": "0.1.6"},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(len(issues), 5)
|
||||||
|
self.assertTrue(any("metadata version '0.1.8' does not match" in issue for issue in issues))
|
||||||
|
self.assertTrue(any("does not publish a govoplan.modules entry point" in issue for issue in issues))
|
||||||
|
self.assertTrue(any("multiple release packages" in issue for issue in issues))
|
||||||
|
|
||||||
|
def test_release_suite_excludes_only_named_source_coupled_tests(self) -> None:
|
||||||
|
retained_id = "tests.test_core_events.CoreEventTests.test_event_delivery"
|
||||||
|
nested = unittest.TestSuite(
|
||||||
|
(
|
||||||
|
unittest.TestSuite(_named_test(test_id) for test_id in SOURCE_COUPLED_CORE_TESTS),
|
||||||
|
unittest.TestSuite((_named_test(retained_id),)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
filtered, removed = filter_test_suite(nested, tuple(SOURCE_COUPLED_CORE_TESTS))
|
||||||
|
|
||||||
|
self.assertEqual(set(removed), set(SOURCE_COUPLED_CORE_TESTS))
|
||||||
|
self.assertEqual([test.id() for test in filtered], [retained_id])
|
||||||
|
self.assertEqual(len(SOURCE_COUPLED_CORE_TESTS), 4)
|
||||||
|
|
||||||
|
def test_release_entrypoint_uses_artifact_checks_and_filtered_core_suite(self) -> None:
|
||||||
|
meta_root = Path(__file__).resolve().parents[1]
|
||||||
|
script = (meta_root / "tools" / "checks" / "check-release-integration.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
artifact_check = script.index('"$META_ROOT/tools/checks/release_integration.py" artifacts')
|
||||||
|
core_tests = script.index('"$META_ROOT/tools/checks/release_integration.py" core-tests')
|
||||||
|
module_tests = script.index('run_step "Run cloned module backend tests"')
|
||||||
|
|
||||||
|
self.assertLess(artifact_check, core_tests)
|
||||||
|
self.assertLess(core_tests, module_tests)
|
||||||
|
self.assertNotIn('"$PYTHON" -m unittest \\\n tests.test_module_system', script)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
126
tests/test_release_sbom.py
Normal file
126
tests/test_release_sbom.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||||
|
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||||
|
|
||||||
|
from govoplan_release.sbom import build_release_sbom, render_sbom, sbom_sha256, validate_sbom # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseSbomTests(unittest.TestCase):
|
||||||
|
@staticmethod
|
||||||
|
def _build_at(timestamp: datetime) -> dict:
|
||||||
|
return build_release_sbom(
|
||||||
|
product_version="1.2.3",
|
||||||
|
pip_inspect={
|
||||||
|
"version": "1",
|
||||||
|
"installed": [{"metadata": {"name": "FastAPI", "version": "0.100.0"}}],
|
||||||
|
},
|
||||||
|
npm_lock={"packages": {"": {"name": "govoplan", "version": "1.2.3"}}},
|
||||||
|
timestamp=timestamp,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_valid_deduplicated_python_and_npm_components(self) -> None:
|
||||||
|
integrity = base64.b64encode(b"example digest").decode("ascii")
|
||||||
|
payload = build_release_sbom(
|
||||||
|
product_version="1.2.3",
|
||||||
|
pip_inspect={
|
||||||
|
"version": "1",
|
||||||
|
"installed": [
|
||||||
|
{"metadata": {"name": "FastAPI", "version": "0.100.0"}},
|
||||||
|
{"metadata": {"name": "FastAPI", "version": "0.100.0"}},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
npm_lock={
|
||||||
|
"packages": {
|
||||||
|
"": {"name": "govoplan", "version": "1.2.3"},
|
||||||
|
"node_modules/@scope/example": {
|
||||||
|
"name": "@scope/example",
|
||||||
|
"version": "4.5.6",
|
||||||
|
"integrity": f"sha512-{integrity}",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
validate_sbom(payload)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(payload["components"]))
|
||||||
|
npm = next(item for item in payload["components"] if item["name"] == "@scope/example")
|
||||||
|
self.assertEqual("SHA-512", npm["hashes"][0]["alg"])
|
||||||
|
self.assertIn("pkg:npm/%40scope/example@4.5.6", npm["purl"])
|
||||||
|
self.assertEqual(
|
||||||
|
hashlib.sha256(render_sbom(payload).encode("utf-8")).hexdigest(),
|
||||||
|
sbom_sha256(payload),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_rejects_empty_component_inventory(self) -> None:
|
||||||
|
payload = build_release_sbom(
|
||||||
|
product_version="1.2.3",
|
||||||
|
pip_inspect={"version": "1", "installed": []},
|
||||||
|
npm_lock={"packages": {"": {"version": "1.2.3"}}},
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "no dependency components"):
|
||||||
|
validate_sbom(payload)
|
||||||
|
|
||||||
|
def test_fixed_timestamp_produces_identical_bytes_and_digest(self) -> None:
|
||||||
|
timestamp = datetime(2026, 7, 21, 8, 30, tzinfo=UTC)
|
||||||
|
|
||||||
|
first = self._build_at(timestamp)
|
||||||
|
second = self._build_at(timestamp)
|
||||||
|
|
||||||
|
self.assertEqual(render_sbom(first), render_sbom(second))
|
||||||
|
self.assertEqual(sbom_sha256(first), sbom_sha256(second))
|
||||||
|
|
||||||
|
def test_timestamp_changes_serial_and_digest(self) -> None:
|
||||||
|
first = self._build_at(datetime(2026, 7, 21, 8, 30, tzinfo=UTC))
|
||||||
|
second = self._build_at(datetime(2026, 7, 21, 8, 31, tzinfo=UTC))
|
||||||
|
|
||||||
|
self.assertNotEqual(first["serialNumber"], second["serialNumber"])
|
||||||
|
self.assertNotEqual(sbom_sha256(first), sbom_sha256(second))
|
||||||
|
|
||||||
|
def test_local_and_credential_bearing_distribution_urls_are_omitted(self) -> None:
|
||||||
|
payload = build_release_sbom(
|
||||||
|
product_version="1.2.3",
|
||||||
|
pip_inspect={
|
||||||
|
"version": "1",
|
||||||
|
"installed": [
|
||||||
|
{
|
||||||
|
"metadata": {"name": "LocalPackage", "version": "1.0.0"},
|
||||||
|
"direct_url": {"url": "file:///private/build/govoplan-core"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metadata": {"name": "PrivatePackage", "version": "1.0.0"},
|
||||||
|
"direct_url": {"url": "https://user:secret@example.test/private.whl"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"metadata": {"name": "PublicPackage", "version": "1.0.0"},
|
||||||
|
"direct_url": {"url": "https://example.test/public.whl"},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
npm_lock={"packages": {"": {"name": "govoplan", "version": "1.2.3"}}},
|
||||||
|
timestamp=datetime(2026, 7, 21, 8, 30, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
|
||||||
|
by_name = {component["name"]: component for component in payload["components"]}
|
||||||
|
self.assertNotIn("externalReferences", by_name["LocalPackage"])
|
||||||
|
self.assertNotIn("externalReferences", by_name["PrivatePackage"])
|
||||||
|
self.assertEqual(
|
||||||
|
[{"type": "distribution", "url": "https://example.test/public.whl"}],
|
||||||
|
by_name["PublicPackage"]["externalReferences"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
358
tests/test_security_audit_wrapper.py
Normal file
358
tests/test_security_audit_wrapper.py
Normal file
@@ -0,0 +1,358 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import textwrap
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
AUDIT_SCRIPT = META_ROOT / "tools" / "checks" / "check-security-audit.sh"
|
||||||
|
|
||||||
|
|
||||||
|
class SecurityAuditWrapperTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self._temporary_directory = tempfile.TemporaryDirectory()
|
||||||
|
self.temporary_root = Path(self._temporary_directory.name)
|
||||||
|
self.checkout = self.temporary_root / "checkout"
|
||||||
|
checks = self.checkout / "tools" / "checks"
|
||||||
|
checks.mkdir(parents=True)
|
||||||
|
shutil.copy2(AUDIT_SCRIPT, checks / AUDIT_SCRIPT.name)
|
||||||
|
(self.checkout / "README.md").write_text("audit fixture\n", encoding="utf-8")
|
||||||
|
|
||||||
|
subprocess.run(["git", "init", "-q", str(self.checkout)], check=True)
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"-C",
|
||||||
|
str(self.checkout),
|
||||||
|
"config",
|
||||||
|
"user.email",
|
||||||
|
"audit@example.invalid",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", str(self.checkout), "config", "user.name", "Audit Test"],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
subprocess.run(["git", "-C", str(self.checkout), "add", "."], check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", str(self.checkout), "commit", "-qm", "audit fixture"],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.stub_bin = self.temporary_root / "bin"
|
||||||
|
self.stub_bin.mkdir()
|
||||||
|
self._write_stub(
|
||||||
|
"semgrep",
|
||||||
|
r"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo 'semgrep stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [[ " $* " != *" --error "* ]]; then
|
||||||
|
echo 'missing --error' >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
output=''
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
if [[ "$1" == '--output' ]]; then
|
||||||
|
output="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
case "${STUB_SEMGREP_REPORT:-valid}" in
|
||||||
|
valid)
|
||||||
|
printf '%s\n' '{"version":"2.1.0","runs":[]}' > "$output"
|
||||||
|
;;
|
||||||
|
malformed)
|
||||||
|
printf '%s\n' '{' > "$output"
|
||||||
|
;;
|
||||||
|
invalid-sarif)
|
||||||
|
printf '%s\n' '{}' > "$output"
|
||||||
|
;;
|
||||||
|
missing)
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if [[ -n "${STUB_MUTATE_PATH:-}" ]]; then
|
||||||
|
printf '%s\n' 'changed during audit' >> "$STUB_MUTATE_PATH"
|
||||||
|
fi
|
||||||
|
exit "${STUB_SEMGREP_EXIT:-0}"
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
self._write_stub(
|
||||||
|
"gitleaks",
|
||||||
|
r"""
|
||||||
|
if [[ "${1:-}" == '--version' || "${1:-}" == 'version' ]]; then
|
||||||
|
echo 'gitleaks stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [[ "${1:-}" == 'git' && "${2:-}" == '--help' ]]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
output=''
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
if [[ "$1" == '--report-path' ]]; then
|
||||||
|
output="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
mkdir -p "$(dirname "$output")"
|
||||||
|
printf '%s\n' '[]' > "$output"
|
||||||
|
exit "${STUB_GITLEAKS_EXIT:-0}"
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
for tool in ("bandit", "ruff"):
|
||||||
|
self._write_stub(
|
||||||
|
tool,
|
||||||
|
f"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo '{tool} stub 1.0'
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self._temporary_directory.cleanup()
|
||||||
|
|
||||||
|
def _write_stub(self, name: str, body: str) -> None:
|
||||||
|
path = self.stub_bin / name
|
||||||
|
path.write_text(
|
||||||
|
"#!/usr/bin/env bash\nset -euo pipefail\n" + textwrap.dedent(body).lstrip(),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
path.chmod(0o755)
|
||||||
|
|
||||||
|
def _install_full_mode_stubs(self) -> None:
|
||||||
|
self._write_stub(
|
||||||
|
"trivy",
|
||||||
|
r"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo 'trivy stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
output=''
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
if [[ "$1" == '--output' ]]; then
|
||||||
|
output="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf '%s\n' '{"version":"2.1.0","runs":[]}' > "$output"
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
self._write_stub(
|
||||||
|
"osv-scanner",
|
||||||
|
r"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo 'osv-scanner stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [[ " $* " == *" scan --help "* ]]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
output=''
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
if [[ "$1" == '--output-file' ]]; then
|
||||||
|
output="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf '{"results": %s}\n' "${STUB_OSV_RESULTS:-null}" > "$output"
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
self._write_stub(
|
||||||
|
"jscpd",
|
||||||
|
r"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo 'jscpd stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
output=''
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
if [[ "$1" == '--output' ]]; then
|
||||||
|
output="$2"
|
||||||
|
shift 2
|
||||||
|
else
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
mkdir -p "$output"
|
||||||
|
printf '%s\n' '{"duplicates":[],"statistics":{}}' > "$output/jscpd-report.json"
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
for tool in ("pip-audit", "npm", "radon", "xenon"):
|
||||||
|
self._write_stub(
|
||||||
|
tool,
|
||||||
|
f"""
|
||||||
|
if [[ " $* " == *" --version "* ]]; then
|
||||||
|
echo '{tool} stub 1.0'
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self, *arguments: str, mode: str = "quick", **environment: str
|
||||||
|
) -> tuple[subprocess.CompletedProcess[str], Path]:
|
||||||
|
reports = self.temporary_root / "reports"
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update(
|
||||||
|
{
|
||||||
|
"PATH": f"{self.stub_bin}:{env['PATH']}",
|
||||||
|
"SECURITY_AUDIT_REQUIRE_TOOLS": "1",
|
||||||
|
"SECURITY_AUDIT_TOOLBOX_FINGERPRINT": 'stub-"safe',
|
||||||
|
**environment,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"bash",
|
||||||
|
str(self.checkout / "tools" / "checks" / AUDIT_SCRIPT.name),
|
||||||
|
"--mode",
|
||||||
|
mode,
|
||||||
|
"--scope",
|
||||||
|
"current",
|
||||||
|
"--reports-dir",
|
||||||
|
str(reports),
|
||||||
|
*arguments,
|
||||||
|
],
|
||||||
|
cwd=self.checkout,
|
||||||
|
env=env,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
return result, reports
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _manifest(reports: Path) -> dict[str, object]:
|
||||||
|
return json.loads((reports / "manifest.json").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def test_report_only_findings_are_nonfatal_and_strict_findings_fail(self) -> None:
|
||||||
|
result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="1")
|
||||||
|
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr)
|
||||||
|
manifest = self._manifest(reports)
|
||||||
|
self.assertEqual(0, manifest["overall_status"])
|
||||||
|
self.assertEqual(1, manifest["finding_status"])
|
||||||
|
self.assertEqual(0, manifest["execution_error_status"])
|
||||||
|
self.assertIn(
|
||||||
|
"Semgrep SAST\t1\tfindings", (reports / "step-status.tsv").read_text()
|
||||||
|
)
|
||||||
|
|
||||||
|
strict_result, strict_reports = self._run("--strict", STUB_SEMGREP_EXIT="1")
|
||||||
|
self.assertEqual(1, strict_result.returncode)
|
||||||
|
self.assertEqual(1, self._manifest(strict_reports)["overall_status"])
|
||||||
|
|
||||||
|
def test_scanner_execution_error_is_fatal_in_report_only_mode(self) -> None:
|
||||||
|
result, reports = self._run("--report-only", STUB_SEMGREP_EXIT="2")
|
||||||
|
|
||||||
|
self.assertEqual(1, result.returncode)
|
||||||
|
manifest = self._manifest(reports)
|
||||||
|
self.assertEqual(1, manifest["overall_status"])
|
||||||
|
self.assertEqual(1, manifest["execution_error_status"])
|
||||||
|
self.assertIn(
|
||||||
|
"Semgrep SAST\t2\texecution-error",
|
||||||
|
(reports / "step-status.tsv").read_text(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_malformed_or_missing_machine_report_is_fatal(self) -> None:
|
||||||
|
for report_kind in ("malformed", "invalid-sarif", "missing"):
|
||||||
|
with self.subTest(report_kind=report_kind):
|
||||||
|
result, reports = self._run(
|
||||||
|
"--report-only",
|
||||||
|
STUB_SEMGREP_REPORT=report_kind,
|
||||||
|
)
|
||||||
|
self.assertEqual(1, result.returncode)
|
||||||
|
manifest = self._manifest(reports)
|
||||||
|
self.assertEqual(1, manifest["execution_error_status"])
|
||||||
|
if report_kind == "missing":
|
||||||
|
self.assertIn("semgrep.sarif", manifest["missing_reports"])
|
||||||
|
self.assertIn(
|
||||||
|
"Validate machine-readable audit reports\t2\texecution-error",
|
||||||
|
(reports / "step-status.tsv").read_text(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_osv_null_results_are_valid_but_wrong_shapes_fail(self) -> None:
|
||||||
|
self._install_full_mode_stubs()
|
||||||
|
|
||||||
|
result, reports = self._run("--report-only", mode="full")
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr)
|
||||||
|
osv_report = json.loads(
|
||||||
|
(reports / "osv-scanner-checkout.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
self.assertIsNone(osv_report["results"])
|
||||||
|
|
||||||
|
invalid_result, invalid_reports = self._run(
|
||||||
|
"--report-only",
|
||||||
|
mode="full",
|
||||||
|
STUB_OSV_RESULTS="{}",
|
||||||
|
)
|
||||||
|
self.assertEqual(1, invalid_result.returncode)
|
||||||
|
self.assertIn(
|
||||||
|
"invalid OSV-Scanner report",
|
||||||
|
invalid_result.stderr,
|
||||||
|
)
|
||||||
|
self.assertEqual(1, self._manifest(invalid_reports)["execution_error_status"])
|
||||||
|
|
||||||
|
def test_workspace_mutation_invalidates_the_audit(self) -> None:
|
||||||
|
result, reports = self._run(
|
||||||
|
"--report-only",
|
||||||
|
STUB_MUTATE_PATH=str(self.checkout / "README.md"),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(1, result.returncode)
|
||||||
|
manifest = self._manifest(reports)
|
||||||
|
self.assertFalse(manifest["workspace_unchanged"])
|
||||||
|
self.assertEqual(1, manifest["overall_status"])
|
||||||
|
self.assertNotEqual(
|
||||||
|
(reports / "workspace-state-start.tsv").read_text(),
|
||||||
|
(reports / "workspace-state-end.tsv").read_text(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_manifest_and_checksums_cover_only_current_run_reports(self) -> None:
|
||||||
|
reports = self.temporary_root / "reports"
|
||||||
|
reports.mkdir()
|
||||||
|
(reports / "stale-malformed.json").write_text("{", encoding="utf-8")
|
||||||
|
|
||||||
|
result, reports = self._run("--report-only")
|
||||||
|
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr)
|
||||||
|
manifest = self._manifest(reports)
|
||||||
|
self.assertEqual('stub-"safe', manifest["toolbox_fingerprint"])
|
||||||
|
self.assertNotIn("stale-malformed.json", manifest["reports"])
|
||||||
|
self.assertEqual([], manifest["missing_reports"])
|
||||||
|
checksums = (reports / "report-sha256.txt").read_text(encoding="utf-8")
|
||||||
|
self.assertNotIn("stale-malformed.json", checksums)
|
||||||
|
verification = subprocess.run(
|
||||||
|
["sha256sum", "--check", "report-sha256.txt"],
|
||||||
|
cwd=reports,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(0, verification.returncode, verification.stderr)
|
||||||
|
checksummed_paths = {
|
||||||
|
line.split(" ", 1)[1] for line in checksums.splitlines() if " " in line
|
||||||
|
}
|
||||||
|
self.assertEqual(set(manifest["reports"]), checksummed_paths)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
225
tests/test_version_alignment.py
Normal file
225
tests/test_version_alignment.py
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
RELEASE_TOOLS_ROOT = META_ROOT / "tools" / "release"
|
||||||
|
if str(RELEASE_TOOLS_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(RELEASE_TOOLS_ROOT))
|
||||||
|
|
||||||
|
from govoplan_release.version_alignment import ( # noqa: E402
|
||||||
|
candidate_catalog_version_issues,
|
||||||
|
release_composition_issues,
|
||||||
|
repository_version_issues,
|
||||||
|
selected_repository_version_issues,
|
||||||
|
)
|
||||||
|
from govoplan_release.model import RepositorySnapshot, RepositorySpec, VersionSnapshot # noqa: E402
|
||||||
|
from govoplan_release.selective_planner import build_unit # noqa: E402
|
||||||
|
from govoplan_release.selective_catalog import enforce_selected_version_alignment # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class VersionAlignmentTests(unittest.TestCase):
|
||||||
|
def test_candidate_catalog_refs_and_selected_units_must_match_versions(self) -> None:
|
||||||
|
payload = {
|
||||||
|
"core_release": {
|
||||||
|
"version": "1.2.3",
|
||||||
|
"python_ref": "govoplan-core @ git+ssh://git@example.test/acme/govoplan-core.git@v1.2.3",
|
||||||
|
},
|
||||||
|
"modules": [
|
||||||
|
{
|
||||||
|
"module_id": "files",
|
||||||
|
"version": "1.2.3",
|
||||||
|
"python_ref": "govoplan-files @ git+ssh://git@example.test/acme/govoplan-files.git@v1.2.4",
|
||||||
|
"webui_package": "@govoplan/files-webui",
|
||||||
|
"webui_ref": "git+ssh://git@example.test/acme/govoplan-files.git#v1.2.3",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"release": {
|
||||||
|
"selected_units": [
|
||||||
|
{"repo": "govoplan-files", "version": "1.2.5", "tag": "v1.2.6"},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
issues = candidate_catalog_version_issues(payload)
|
||||||
|
|
||||||
|
messages = {issue.message for issue in issues}
|
||||||
|
self.assertIn("Python ref tag must match the catalog entry version", messages)
|
||||||
|
self.assertIn("selected-unit tag must match its version", messages)
|
||||||
|
self.assertIn("selected-unit version must match the catalog entry", messages)
|
||||||
|
|
||||||
|
def test_catalog_candidate_gate_rejects_selected_repo_drift(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
workspace = Path(tmp)
|
||||||
|
repo = workspace / "govoplan-files"
|
||||||
|
(repo / "webui").mkdir(parents=True)
|
||||||
|
(repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n')
|
||||||
|
(repo / "webui" / "package.json").write_text('{"version":"1.2.4"}\n')
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "Version alignment gate failed"):
|
||||||
|
enforce_selected_version_alignment(
|
||||||
|
repo_versions={"govoplan-files": "1.2.5"},
|
||||||
|
workspace=workspace,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_catalog_candidate_gate_rejects_requested_version_that_differs_from_source(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
workspace = Path(tmp)
|
||||||
|
repo = workspace / "govoplan-files"
|
||||||
|
(repo / "webui").mkdir(parents=True)
|
||||||
|
(repo / "pyproject.toml").write_text('[project]\nname="govoplan-files"\nversion="1.2.3"\n')
|
||||||
|
(repo / "webui" / "package.json").write_text('{"version":"1.2.3"}\n')
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "source version must match the requested release version"):
|
||||||
|
enforce_selected_version_alignment(
|
||||||
|
repo_versions={"govoplan-files": "1.2.4"},
|
||||||
|
workspace=workspace,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_selected_repository_gate_reports_missing_version_metadata(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
workspace = Path(tmp)
|
||||||
|
(workspace / "govoplan-files").mkdir()
|
||||||
|
|
||||||
|
issues = selected_repository_version_issues(
|
||||||
|
repo_versions={"govoplan-files": "1.2.3"},
|
||||||
|
workspace=workspace,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(1, len(issues))
|
||||||
|
self.assertEqual("repository has no version metadata", issues[0].message)
|
||||||
|
|
||||||
|
def test_selective_release_unit_is_blocked_by_version_drift(self) -> None:
|
||||||
|
repo = RepositorySnapshot(
|
||||||
|
spec=RepositorySpec(
|
||||||
|
name="govoplan-example",
|
||||||
|
category="module",
|
||||||
|
subtype="domain",
|
||||||
|
remote="git@example.test:acme/govoplan-example.git",
|
||||||
|
path="govoplan-example",
|
||||||
|
),
|
||||||
|
absolute_path="/tmp/govoplan-example",
|
||||||
|
exists=True,
|
||||||
|
is_git=True,
|
||||||
|
has_head=True,
|
||||||
|
versions=VersionSnapshot(pyproject="1.2.3", webui_package="1.2.4", manifests=("1.2.3",)),
|
||||||
|
)
|
||||||
|
|
||||||
|
unit = build_unit(repo, target_version="1.2.5", contracts=None)
|
||||||
|
|
||||||
|
self.assertEqual("blocked", unit.status)
|
||||||
|
self.assertIn("version metadata is not aligned", unit.blockers[0])
|
||||||
|
|
||||||
|
def test_repository_versions_and_lockfiles_must_match(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp) / "govoplan-example"
|
||||||
|
(root / "src" / "govoplan_example" / "backend").mkdir(parents=True)
|
||||||
|
(root / "webui").mkdir()
|
||||||
|
(root / "pyproject.toml").write_text('[project]\nname = "govoplan-example"\nversion = "1.2.3"\n')
|
||||||
|
(root / "package.json").write_text('{"name":"@govoplan/example","version":"1.2.3"}\n')
|
||||||
|
(root / "webui" / "package.json").write_text('{"name":"@govoplan/example-webui","version":"1.2.4"}\n')
|
||||||
|
(root / "webui" / "package-lock.json").write_text(
|
||||||
|
json.dumps({"packages": {"": {"version": "1.2.2"}}})
|
||||||
|
)
|
||||||
|
(root / "src" / "govoplan_example" / "backend" / "manifest.py").write_text(
|
||||||
|
'manifest = ModuleManifest(\n version="1.2.3",\n)\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = repository_version_issues(root)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(issues))
|
||||||
|
self.assertEqual({"webui/package.json", "webui/package-lock.json"}, {issue.source for issue in issues})
|
||||||
|
|
||||||
|
def test_public_runtime_version_must_match_package_metadata(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp) / "govoplan-example"
|
||||||
|
package = root / "src" / "govoplan_example"
|
||||||
|
package.mkdir(parents=True)
|
||||||
|
(root / "pyproject.toml").write_text(
|
||||||
|
'[project]\nname = "govoplan-example"\nversion = "1.2.3"\n'
|
||||||
|
)
|
||||||
|
(package / "__init__.py").write_text('__version__ = "1.2.2"\n')
|
||||||
|
|
||||||
|
issues = repository_version_issues(root)
|
||||||
|
|
||||||
|
self.assertEqual(1, len(issues))
|
||||||
|
self.assertEqual("package __init__.py 1", issues[0].source)
|
||||||
|
self.assertEqual("1.2.3", issues[0].expected)
|
||||||
|
self.assertEqual("1.2.2", issues[0].actual)
|
||||||
|
|
||||||
|
def test_release_backend_and_frontend_refs_must_match(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
meta = Path(tmp) / "govoplan"
|
||||||
|
core = Path(tmp) / "govoplan-core"
|
||||||
|
meta.mkdir()
|
||||||
|
(core / "webui").mkdir(parents=True)
|
||||||
|
(meta / "requirements-release.txt").write_text(
|
||||||
|
"govoplan-example @ git+ssh://git@example.test/acme/govoplan-example.git@v1.2.3\n"
|
||||||
|
)
|
||||||
|
(core / "webui" / "package.release.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"@govoplan/example-webui": (
|
||||||
|
"git+ssh://git@example.test/acme/govoplan-example.git#v1.2.4"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = release_composition_issues(meta, core_root=core)
|
||||||
|
|
||||||
|
self.assertEqual(1, len(issues))
|
||||||
|
self.assertEqual("1.2.3", issues[0].expected)
|
||||||
|
self.assertEqual("1.2.4", issues[0].actual)
|
||||||
|
|
||||||
|
def test_release_refs_must_point_at_installable_versioned_artifacts(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
workspace = Path(tmp)
|
||||||
|
meta = workspace / "govoplan"
|
||||||
|
core = workspace / "govoplan-core"
|
||||||
|
module = workspace / "govoplan-example"
|
||||||
|
meta.mkdir()
|
||||||
|
(core / "webui").mkdir(parents=True)
|
||||||
|
module.mkdir()
|
||||||
|
(meta / "requirements-release.txt").write_text(
|
||||||
|
"govoplan-example @ git+ssh://git@example.test/acme/govoplan-example.git@v1.2.3\n"
|
||||||
|
)
|
||||||
|
(core / "webui" / "package.release.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"@govoplan/example-webui": (
|
||||||
|
"git+ssh://git@example.test/acme/govoplan-example.git#v1.2.3"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
subprocess.run(["git", "init", "-q", str(module)], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(module), "config", "user.email", "test@example.test"], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(module), "config", "user.name", "Test"], check=True)
|
||||||
|
(module / "README.md").write_text("uninstallable release\n")
|
||||||
|
subprocess.run(["git", "-C", str(module), "add", "README.md"], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(module), "commit", "-qm", "release"], check=True)
|
||||||
|
subprocess.run(["git", "-C", str(module), "tag", "v1.2.3"], check=True)
|
||||||
|
|
||||||
|
issues = release_composition_issues(meta, core_root=core)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(issues))
|
||||||
|
self.assertEqual(
|
||||||
|
{"v1.2.3:pyproject.toml", "v1.2.3:webui/package.json"},
|
||||||
|
{issue.source for issue in issues},
|
||||||
|
)
|
||||||
|
self.assertTrue(all("must contain" in issue.message for issue in issues))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -32,9 +32,8 @@ with (Path.cwd() / "pyproject.toml").open("rb") as handle:
|
|||||||
print(tomllib.load(handle)["project"]["version"])
|
print(tomllib.load(handle)["project"]["version"])
|
||||||
PY
|
PY
|
||||||
)"
|
)"
|
||||||
RELEASE_TAG="${GOVOPLAN_RELEASE_TAG:-v$RELEASE_VERSION}"
|
|
||||||
|
|
||||||
export RELEASE_TAG RELEASE_VERSION
|
export RELEASE_VERSION
|
||||||
export META_ROOT
|
export META_ROOT
|
||||||
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
|
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
|
||||||
export GOVOPLAN_MIGRATION_TRACK=release
|
export GOVOPLAN_MIGRATION_TRACK=release
|
||||||
@@ -91,7 +90,32 @@ install_cloned_webui_dependencies() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
release_tag_for_package() {
|
||||||
|
local package_name="$1"
|
||||||
|
"$PYTHON" - "$META_ROOT/requirements-release.txt" "$package_name" <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
requirements_path = Path(sys.argv[1])
|
||||||
|
package_name = sys.argv[2]
|
||||||
|
pattern = re.compile(
|
||||||
|
rf"^{re.escape(package_name)}\s+@\s+git\+ssh://.+?\.git@(v[0-9]+\.[0-9]+\.[0-9]+)$"
|
||||||
|
)
|
||||||
|
for line in requirements_path.read_text(encoding="utf-8").splitlines():
|
||||||
|
match = pattern.match(line.strip())
|
||||||
|
if match:
|
||||||
|
print(match.group(1))
|
||||||
|
raise SystemExit(0)
|
||||||
|
raise SystemExit(f"No immutable release reference found for {package_name}")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
run_step "Validate release refs and installed package metadata"
|
run_step "Validate release refs and installed package metadata"
|
||||||
|
"$PYTHON" "$META_ROOT/tools/checks/check-version-alignment.py" --release-composition
|
||||||
"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact
|
"$PYTHON" "$META_ROOT/tools/checks/check-contracts.py" --no-impact
|
||||||
"$PYTHON" - <<'PY'
|
"$PYTHON" - <<'PY'
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -108,7 +132,6 @@ root = Path.cwd()
|
|||||||
release_requirements = Path(os.environ["META_ROOT"]) / "requirements-release.txt"
|
release_requirements = Path(os.environ["META_ROOT"]) / "requirements-release.txt"
|
||||||
release_webui = root / "webui" / "package.release.json"
|
release_webui = root / "webui" / "package.release.json"
|
||||||
expected_version = os.environ["RELEASE_VERSION"]
|
expected_version = os.environ["RELEASE_VERSION"]
|
||||||
expected_tag = os.environ["RELEASE_TAG"]
|
|
||||||
errors: list[str] = []
|
errors: list[str] = []
|
||||||
|
|
||||||
python_requirements: list[tuple[str, str, str]] = []
|
python_requirements: list[tuple[str, str, str]] = []
|
||||||
@@ -121,8 +144,6 @@ if not python_requirements:
|
|||||||
errors.append("No GovOPlaN git+ssh release requirements found.")
|
errors.append("No GovOPlaN git+ssh release requirements found.")
|
||||||
|
|
||||||
for package_name, repo_url, tag in python_requirements:
|
for package_name, repo_url, tag in python_requirements:
|
||||||
if tag != expected_tag:
|
|
||||||
errors.append(f"{package_name} release requirement uses {tag!r}, expected {expected_tag!r}.")
|
|
||||||
version = metadata.version(package_name)
|
version = metadata.version(package_name)
|
||||||
if version != tag.removeprefix("v"):
|
if version != tag.removeprefix("v"):
|
||||||
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
|
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
|
||||||
@@ -146,15 +167,18 @@ for package_name, spec in sorted(webui_dependencies.items()):
|
|||||||
if not package_name.startswith("@govoplan/"):
|
if not package_name.startswith("@govoplan/"):
|
||||||
continue
|
continue
|
||||||
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
|
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
|
||||||
if tag != expected_tag:
|
tag_match = re.fullmatch(r"v([0-9]+\.[0-9]+\.[0-9]+)", tag)
|
||||||
errors.append(f"{package_name} release dependency uses {tag!r}, expected {expected_tag}.")
|
if tag_match is None:
|
||||||
|
errors.append(f"{package_name} release dependency has invalid immutable tag {tag!r}.")
|
||||||
|
continue
|
||||||
|
dependency_version = tag_match.group(1)
|
||||||
package_json = root / "webui" / "node_modules" / package_name / "package.json"
|
package_json = root / "webui" / "node_modules" / package_name / "package.json"
|
||||||
if not package_json.exists():
|
if not package_json.exists():
|
||||||
errors.append(f"{package_name} is missing from release node_modules.")
|
errors.append(f"{package_name} is missing from release node_modules.")
|
||||||
continue
|
continue
|
||||||
installed = json.loads(package_json.read_text(encoding="utf-8"))
|
installed = json.loads(package_json.read_text(encoding="utf-8"))
|
||||||
if installed.get("version") != expected_version:
|
if installed.get("version") != dependency_version:
|
||||||
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {expected_version!r}.")
|
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {dependency_version!r} from {tag!r}.")
|
||||||
|
|
||||||
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
|
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
|
||||||
root_peers = release_package.get("peerDependencies", {})
|
root_peers = release_package.get("peerDependencies", {})
|
||||||
@@ -173,40 +197,14 @@ print(f"Release refs and metadata passed for {len(python_requirements)} Python p
|
|||||||
PY
|
PY
|
||||||
|
|
||||||
run_step "Validate installed module manifests and registry"
|
run_step "Validate installed module manifests and registry"
|
||||||
"$PYTHON" - <<'PY'
|
"$PYTHON" "$META_ROOT/tools/checks/release_integration.py" artifacts \
|
||||||
from __future__ import annotations
|
--requirements "$META_ROOT/requirements-release.txt"
|
||||||
|
|
||||||
import sys
|
run_step "Generate release dependency provenance"
|
||||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
"$PYTHON" "$META_ROOT/tools/release/generate-release-sbom.py" \
|
||||||
|
--python "$PYTHON" \
|
||||||
expected = {
|
--core-root "$ROOT" \
|
||||||
"access",
|
--output "$WORK_ROOT/govoplan-sbom.cdx.json"
|
||||||
"admin",
|
|
||||||
"audit",
|
|
||||||
"calendar",
|
|
||||||
"campaigns",
|
|
||||||
"dashboard",
|
|
||||||
"docs",
|
|
||||||
"files",
|
|
||||||
"identity",
|
|
||||||
"idm",
|
|
||||||
"mail",
|
|
||||||
"ops",
|
|
||||||
"organizations",
|
|
||||||
"policy",
|
|
||||||
"tenancy",
|
|
||||||
}
|
|
||||||
|
|
||||||
manifests = available_module_manifests()
|
|
||||||
missing = sorted(expected - set(manifests))
|
|
||||||
if missing:
|
|
||||||
print(f"Missing installed module manifests: {', '.join(missing)}", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
registry = build_platform_registry(tuple(sorted(expected)))
|
|
||||||
snapshot = registry.validate()
|
|
||||||
print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests))
|
|
||||||
PY
|
|
||||||
|
|
||||||
run_step "Run SQLite migration smoke for release modules"
|
run_step "Run SQLite migration smoke for release modules"
|
||||||
"$PYTHON" - <<'PY'
|
"$PYTHON" - <<'PY'
|
||||||
@@ -245,16 +243,13 @@ PY
|
|||||||
|
|
||||||
run_step "Clone release module test sources"
|
run_step "Clone release module test sources"
|
||||||
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
|
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
|
||||||
git clone --depth 1 --branch "$RELEASE_TAG" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
repo_tag="$(release_tag_for_package "$repo")"
|
||||||
|
git clone --depth 1 --branch "$repo_tag" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
||||||
done
|
done
|
||||||
|
|
||||||
run_step "Run core backend release tests"
|
run_step "Run core backend release tests"
|
||||||
"$PYTHON" -m unittest \
|
"$PYTHON" "$META_ROOT/tools/checks/release_integration.py" core-tests \
|
||||||
tests.test_module_system \
|
--core-root "$ROOT"
|
||||||
tests.test_access_contracts \
|
|
||||||
tests.test_identity_organization_contracts \
|
|
||||||
tests.test_core_events \
|
|
||||||
tests.test_policy_contracts
|
|
||||||
"$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py"
|
"$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py"
|
||||||
|
|
||||||
run_step "Run cloned module backend tests"
|
run_step "Run cloned module backend tests"
|
||||||
|
|||||||
@@ -77,6 +77,15 @@ if [[ "$REPORTS_DIR" != /* ]]; then
|
|||||||
REPORTS_DIR="$ROOT/$REPORTS_DIR"
|
REPORTS_DIR="$ROOT/$REPORTS_DIR"
|
||||||
fi
|
fi
|
||||||
mkdir -p "$REPORTS_DIR"
|
mkdir -p "$REPORTS_DIR"
|
||||||
|
REPORTS_DIR="$(cd "$REPORTS_DIR" && pwd -P)"
|
||||||
|
|
||||||
|
for flag_name in FAIL_ON_FINDINGS REQUIRE_TOOLS; do
|
||||||
|
flag_value="${!flag_name}"
|
||||||
|
if [[ "$flag_value" != "0" && "$flag_value" != "1" ]]; then
|
||||||
|
echo "$flag_name must be 0 or 1, got: $flag_value" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
declare -a REPOS=()
|
declare -a REPOS=()
|
||||||
if [[ "$SCOPE" == "govoplan" ]]; then
|
if [[ "$SCOPE" == "govoplan" ]]; then
|
||||||
@@ -118,7 +127,7 @@ for repo in "${REPOS[@]}"; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
safe_name() {
|
safe_name() {
|
||||||
basename "$1" | tr -c 'A-Za-z0-9_.-' '_'
|
printf '%s' "$(basename "$1")" | tr -c 'A-Za-z0-9_.-' '_'
|
||||||
}
|
}
|
||||||
|
|
||||||
has_tool() {
|
has_tool() {
|
||||||
@@ -128,58 +137,341 @@ has_tool() {
|
|||||||
overall_status=0
|
overall_status=0
|
||||||
finding_status=0
|
finding_status=0
|
||||||
missing_status=0
|
missing_status=0
|
||||||
|
execution_status=0
|
||||||
|
audit_started_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||||
|
audit_completed_at=""
|
||||||
|
workspace_unchanged="unknown"
|
||||||
|
audit_toolbox_fingerprint="${SECURITY_AUDIT_TOOLBOX_FINGERPRINT:-direct-host}"
|
||||||
|
declare -A REPORT_FILES=()
|
||||||
|
declare -A MACHINE_REPORTS=()
|
||||||
|
|
||||||
|
register_report() {
|
||||||
|
local path="$1"
|
||||||
|
local report_kind="${2:-text}"
|
||||||
|
case "$path" in
|
||||||
|
"$REPORTS_DIR"/*) ;;
|
||||||
|
*)
|
||||||
|
echo "Refusing to register an audit report outside $REPORTS_DIR: $path" >&2
|
||||||
|
return 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
REPORT_FILES["$path"]=1
|
||||||
|
if [[ "$report_kind" == "machine" ]]; then
|
||||||
|
MACHINE_REPORTS["$path"]=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_machine_report() {
|
||||||
|
local path="$1"
|
||||||
|
register_report "$path" machine || return 2
|
||||||
|
if [[ -e "$path" || -L "$path" ]]; then
|
||||||
|
rm -f -- "$path" || return 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
run_step() {
|
run_step() {
|
||||||
local name="$1"
|
local name="$1"
|
||||||
shift
|
local exit_contract="$2"
|
||||||
|
shift 2
|
||||||
echo
|
echo
|
||||||
echo "==> $name"
|
echo "==> $name"
|
||||||
"$@"
|
"$@"
|
||||||
local status=$?
|
local status=$?
|
||||||
if [[ "$status" -ne 0 ]]; then
|
local outcome="passed"
|
||||||
echo "Audit step reported findings or failed: $name (exit $status)" >&2
|
if [[ "$status" -eq 0 ]]; then
|
||||||
|
:
|
||||||
|
elif [[ "$exit_contract" == "findings-exit-one" && "$status" -eq 1 ]]; then
|
||||||
|
outcome="findings"
|
||||||
|
echo "Audit step reported findings: $name (exit $status)" >&2
|
||||||
finding_status=1
|
finding_status=1
|
||||||
if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
|
if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
|
||||||
overall_status=1
|
overall_status=1
|
||||||
fi
|
fi
|
||||||
|
else
|
||||||
|
outcome="execution-error"
|
||||||
|
echo "Audit step failed to execute successfully: $name (exit $status)" >&2
|
||||||
|
execution_status=1
|
||||||
|
overall_status=1
|
||||||
fi
|
fi
|
||||||
|
printf '%s\t%s\t%s\n' "$name" "$status" "$outcome" >> "$REPORTS_DIR/step-status.tsv"
|
||||||
}
|
}
|
||||||
|
|
||||||
skip_or_fail_missing() {
|
skip_or_fail_missing() {
|
||||||
local tool="$1"
|
local tool="$1"
|
||||||
echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
|
echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
|
||||||
|
missing_status=1
|
||||||
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
|
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
|
||||||
missing_status=1
|
|
||||||
overall_status=1
|
overall_status=1
|
||||||
fi
|
fi
|
||||||
|
printf '%s\t127\tmissing\n' "$tool" >> "$REPORTS_DIR/step-status.tsv"
|
||||||
}
|
}
|
||||||
|
|
||||||
write_manifest() {
|
write_manifest() {
|
||||||
{
|
local -a present_reports=()
|
||||||
printf '{\n'
|
local -a expected_reports=()
|
||||||
printf ' "mode": "%s",\n' "$MODE"
|
local path
|
||||||
printf ' "scope": "%s",\n' "$SCOPE"
|
while IFS= read -r path; do
|
||||||
printf ' "fail_on_findings": "%s",\n' "$FAIL_ON_FINDINGS"
|
expected_reports+=("${path#"$REPORTS_DIR"/}")
|
||||||
printf ' "repositories": [\n'
|
if [[ -f "$path" ]]; then
|
||||||
local index=0
|
present_reports+=("${path#"$REPORTS_DIR"/}")
|
||||||
for repo in "${REPOS[@]}"; do
|
fi
|
||||||
[[ "$index" -gt 0 ]] && printf ',\n'
|
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
||||||
printf ' "%s"' "$repo"
|
|
||||||
index=$((index + 1))
|
python3 - \
|
||||||
done
|
"$REPORTS_DIR/manifest.json" \
|
||||||
printf '\n ]\n'
|
"$MODE" \
|
||||||
printf '}\n'
|
"$SCOPE" \
|
||||||
} > "$REPORTS_DIR/manifest.json"
|
"$FAIL_ON_FINDINGS" \
|
||||||
|
"$audit_started_at" \
|
||||||
|
"$audit_completed_at" \
|
||||||
|
"$workspace_unchanged" \
|
||||||
|
"$overall_status" \
|
||||||
|
"$finding_status" \
|
||||||
|
"$execution_status" \
|
||||||
|
"$missing_status" \
|
||||||
|
"$audit_toolbox_fingerprint" \
|
||||||
|
"${#REPOS[@]}" \
|
||||||
|
"${REPOS[@]}" \
|
||||||
|
"${#present_reports[@]}" \
|
||||||
|
"${present_reports[@]}" \
|
||||||
|
"${expected_reports[@]}" <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
(
|
||||||
|
manifest_path,
|
||||||
|
mode,
|
||||||
|
scope,
|
||||||
|
fail_on_findings,
|
||||||
|
started_at,
|
||||||
|
completed_at,
|
||||||
|
workspace_unchanged,
|
||||||
|
overall_status,
|
||||||
|
finding_status,
|
||||||
|
execution_status,
|
||||||
|
missing_status,
|
||||||
|
toolbox_fingerprint,
|
||||||
|
repository_count,
|
||||||
|
*remaining,
|
||||||
|
) = sys.argv[1:]
|
||||||
|
repo_count = int(repository_count)
|
||||||
|
repositories = remaining[:repo_count]
|
||||||
|
remaining = remaining[repo_count:]
|
||||||
|
report_count = int(remaining[0])
|
||||||
|
reports = remaining[1 : report_count + 1]
|
||||||
|
expected_reports = remaining[report_count + 1 :]
|
||||||
|
payload = {
|
||||||
|
"mode": mode,
|
||||||
|
"scope": scope,
|
||||||
|
"fail_on_findings": fail_on_findings == "1",
|
||||||
|
"started_at_utc": started_at,
|
||||||
|
"completed_at_utc": completed_at,
|
||||||
|
"toolbox_fingerprint": toolbox_fingerprint,
|
||||||
|
"workspace_unchanged": workspace_unchanged == "true",
|
||||||
|
"overall_status": int(overall_status),
|
||||||
|
"finding_status": int(finding_status),
|
||||||
|
"execution_error_status": int(execution_status),
|
||||||
|
"missing_tool_status": int(missing_status),
|
||||||
|
"tool_versions": "tool-versions.txt",
|
||||||
|
"workspace_state_start": "workspace-state-start.tsv",
|
||||||
|
"workspace_state_end": "workspace-state-end.tsv",
|
||||||
|
"report_checksums": "report-sha256.txt",
|
||||||
|
"step_status": "step-status.tsv",
|
||||||
|
"repositories": repositories,
|
||||||
|
"reports": reports,
|
||||||
|
"expected_reports": expected_reports,
|
||||||
|
"missing_reports": sorted(set(expected_reports) - set(reports)),
|
||||||
|
}
|
||||||
|
Path(manifest_path).write_text(
|
||||||
|
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
accumulate_exit_status() {
|
||||||
|
local -n aggregate="$1"
|
||||||
|
local child_status="$2"
|
||||||
|
if [[ "$child_status" -gt 1 ]]; then
|
||||||
|
aggregate=2
|
||||||
|
elif [[ "$child_status" -eq 1 && "$aggregate" -eq 0 ]]; then
|
||||||
|
aggregate=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
repository_fingerprint() {
|
||||||
|
local repo="$1"
|
||||||
|
(
|
||||||
|
set -e
|
||||||
|
cd "$repo" || exit 1
|
||||||
|
git rev-parse HEAD
|
||||||
|
git status --porcelain=v1 -z
|
||||||
|
git diff --binary --no-ext-diff HEAD --
|
||||||
|
while IFS= read -r -d '' path; do
|
||||||
|
printf '%s\0' "$path"
|
||||||
|
if [[ -L "$path" ]]; then
|
||||||
|
printf 'symlink\0'
|
||||||
|
readlink -- "$path"
|
||||||
|
else
|
||||||
|
sha256sum -- "$path"
|
||||||
|
fi
|
||||||
|
done < <(git ls-files --others --exclude-standard -z)
|
||||||
|
) | sha256sum | cut -d' ' -f1
|
||||||
|
}
|
||||||
|
|
||||||
|
write_workspace_state() {
|
||||||
|
local destination="$1"
|
||||||
|
local head
|
||||||
|
local fingerprint
|
||||||
|
: > "$destination"
|
||||||
|
for repo in "${REPOS[@]}"; do
|
||||||
|
if ! head="$(git -C "$repo" rev-parse HEAD)"; then
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
if ! fingerprint="$(repository_fingerprint "$repo")"; then
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
printf '%s\t%s\t%s\n' \
|
||||||
|
"$repo" \
|
||||||
|
"$head" \
|
||||||
|
"$fingerprint" \
|
||||||
|
>> "$destination"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
write_tool_versions() {
|
||||||
|
local destination="$REPORTS_DIR/tool-versions.txt"
|
||||||
|
local status=0
|
||||||
|
local version
|
||||||
|
: > "$destination"
|
||||||
|
local tool
|
||||||
|
for tool in semgrep bandit ruff gitleaks trivy pip-audit npm osv-scanner jscpd radon xenon; do
|
||||||
|
has_tool "$tool" || continue
|
||||||
|
if [[ "$tool" == "gitleaks" ]]; then
|
||||||
|
version="$("$tool" version 2>&1)"
|
||||||
|
else
|
||||||
|
version="$("$tool" --version 2>&1)"
|
||||||
|
fi
|
||||||
|
local version_status=$?
|
||||||
|
version="${version%%$'\n'*}"
|
||||||
|
if [[ "$version_status" -ne 0 || -z "$version" ]]; then
|
||||||
|
printf '%s\t%s\n' "$tool" "version-command-failed:$version_status" >> "$destination"
|
||||||
|
status=2
|
||||||
|
else
|
||||||
|
printf '%s\t%s\n' "$tool" "$version" >> "$destination"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_report_checksums() {
|
||||||
|
local path
|
||||||
|
local relative_path
|
||||||
|
: > "$REPORTS_DIR/report-sha256.txt"
|
||||||
|
while IFS= read -r path; do
|
||||||
|
[[ -f "$path" ]] || continue
|
||||||
|
relative_path="${path#"$REPORTS_DIR"/}"
|
||||||
|
(cd "$REPORTS_DIR" && sha256sum -- "$relative_path") \
|
||||||
|
>> "$REPORTS_DIR/report-sha256.txt" || return 2
|
||||||
|
done < <(printf '%s\n' "${!REPORT_FILES[@]}" | sort)
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_machine_reports() {
|
||||||
|
python3 - "$REPORTS_DIR" "${!MACHINE_REPORTS[@]}" <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
root = Path(sys.argv[1]).resolve()
|
||||||
|
errors: list[str] = []
|
||||||
|
validated = 0
|
||||||
|
for raw_path in sorted(set(sys.argv[2:])):
|
||||||
|
path = Path(raw_path)
|
||||||
|
try:
|
||||||
|
relative_path = path.resolve().relative_to(root)
|
||||||
|
except (OSError, ValueError) as exc:
|
||||||
|
errors.append(f"{path}: invalid report path: {exc}")
|
||||||
|
continue
|
||||||
|
if not path.is_file():
|
||||||
|
errors.append(f"{relative_path}: expected report was not produced")
|
||||||
|
continue
|
||||||
|
if path.stat().st_size == 0:
|
||||||
|
errors.append(f"{relative_path}: report is empty")
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||||
|
errors.append(f"{relative_path}: invalid JSON: {exc}")
|
||||||
|
continue
|
||||||
|
validated += 1
|
||||||
|
if path.suffix == ".sarif" and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or not isinstance(payload.get("version"), str)
|
||||||
|
or not isinstance(payload.get("runs"), list)
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: invalid SARIF document")
|
||||||
|
continue
|
||||||
|
report_name = path.name
|
||||||
|
if report_name.startswith("bandit") and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or not isinstance(payload.get("errors"), list)
|
||||||
|
or not isinstance(payload.get("metrics"), dict)
|
||||||
|
or not isinstance(payload.get("results"), list)
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: invalid Bandit report")
|
||||||
|
elif report_name.startswith("ruff-security") and not isinstance(payload, list):
|
||||||
|
errors.append(f"{relative_path}: invalid Ruff report")
|
||||||
|
elif report_name.startswith("gitleaks-") and not isinstance(payload, list):
|
||||||
|
errors.append(f"{relative_path}: invalid Gitleaks report")
|
||||||
|
elif report_name.startswith("pip-audit-") and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or not isinstance(payload.get("dependencies"), list)
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: invalid pip-audit report")
|
||||||
|
elif report_name.startswith("npm-audit-") and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or payload.get("error")
|
||||||
|
or not isinstance(payload.get("metadata"), dict)
|
||||||
|
or not isinstance(payload.get("vulnerabilities"), dict)
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: npm audit returned an invalid or error report")
|
||||||
|
elif report_name.startswith("osv-scanner-") and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or "results" not in payload
|
||||||
|
or (payload["results"] is not None and not isinstance(payload["results"], list))
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: invalid OSV-Scanner report")
|
||||||
|
elif report_name == "jscpd-report.json" and (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or not isinstance(payload.get("duplicates"), list)
|
||||||
|
or not isinstance(payload.get("statistics"), dict)
|
||||||
|
):
|
||||||
|
errors.append(f"{relative_path}: invalid jscpd report")
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
print("\n".join(errors), file=sys.stderr)
|
||||||
|
raise SystemExit(2)
|
||||||
|
print(f"Validated {validated} JSON/SARIF reports.")
|
||||||
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
run_semgrep() {
|
run_semgrep() {
|
||||||
local report="$REPORTS_DIR/semgrep.sarif"
|
local report="$REPORTS_DIR/semgrep.sarif"
|
||||||
|
prepare_machine_report "$report" || return 2
|
||||||
local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml")
|
local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml")
|
||||||
if [[ "$MODE" != "quick" ]]; then
|
if [[ "$MODE" != "quick" ]]; then
|
||||||
config_args+=(--config p/default --config p/owasp-top-ten)
|
config_args+=(--config p/default --config p/owasp-top-ten)
|
||||||
fi
|
fi
|
||||||
semgrep scan \
|
semgrep scan \
|
||||||
--metrics=off \
|
--metrics=off \
|
||||||
|
--error \
|
||||||
--no-git-ignore \
|
--no-git-ignore \
|
||||||
--sarif \
|
--sarif \
|
||||||
--output "$report" \
|
--output "$report" \
|
||||||
@@ -198,10 +490,15 @@ run_semgrep() {
|
|||||||
run_bandit() {
|
run_bandit() {
|
||||||
local status=0
|
local status=0
|
||||||
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
||||||
bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json" || status=1
|
prepare_machine_report "$REPORTS_DIR/bandit.json" || return 2
|
||||||
|
bandit -r "${PY_PROD_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
fi
|
fi
|
||||||
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
||||||
bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json" || true
|
prepare_machine_report "$REPORTS_DIR/bandit-tests.json" || return 2
|
||||||
|
bandit -r "${PY_TEST_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit-tests.json"
|
||||||
|
local test_status=$?
|
||||||
|
[[ "$test_status" -le 1 ]] || status=2
|
||||||
fi
|
fi
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
@@ -209,10 +506,15 @@ run_bandit() {
|
|||||||
run_ruff_security() {
|
run_ruff_security() {
|
||||||
local status=0
|
local status=0
|
||||||
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
if [[ "${#PY_PROD_ROOTS[@]}" -gt 0 ]]; then
|
||||||
ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json" || status=1
|
prepare_machine_report "$REPORTS_DIR/ruff-security.json" || return 2
|
||||||
|
ruff check --select S --output-format json "${PY_PROD_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
fi
|
fi
|
||||||
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
if [[ "${#PY_TEST_ROOTS[@]}" -gt 0 ]]; then
|
||||||
ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json" || true
|
prepare_machine_report "$REPORTS_DIR/ruff-security-tests.json" || return 2
|
||||||
|
ruff check --select S --output-format json "${PY_TEST_ROOTS[@]}" > "$REPORTS_DIR/ruff-security-tests.json"
|
||||||
|
local test_status=$?
|
||||||
|
[[ "$test_status" -le 1 ]] || status=2
|
||||||
fi
|
fi
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
@@ -223,12 +525,15 @@ run_gitleaks() {
|
|||||||
local name
|
local name
|
||||||
name="$(safe_name "$repo")"
|
name="$(safe_name "$repo")"
|
||||||
if gitleaks git --help >/dev/null 2>&1; then
|
if gitleaks git --help >/dev/null 2>&1; then
|
||||||
|
prepare_machine_report "$REPORTS_DIR/gitleaks-history-$name.json" || return 2
|
||||||
|
prepare_machine_report "$REPORTS_DIR/gitleaks-worktree-$name.json" || return 2
|
||||||
gitleaks git \
|
gitleaks git \
|
||||||
--config "$ROOT/.gitleaks.toml" \
|
--config "$ROOT/.gitleaks.toml" \
|
||||||
--report-format json \
|
--report-format json \
|
||||||
--report-path "$REPORTS_DIR/gitleaks-history-$name.json" \
|
--report-path "$REPORTS_DIR/gitleaks-history-$name.json" \
|
||||||
--no-banner \
|
--no-banner \
|
||||||
"$repo" || status=1
|
"$repo"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
# History scanning does not include new or modified working-tree files.
|
# History scanning does not include new or modified working-tree files.
|
||||||
# Scan the directory as well so pre-commit audits cover the exact code
|
# Scan the directory as well so pre-commit audits cover the exact code
|
||||||
# under review, while retaining the history scan above.
|
# under review, while retaining the history scan above.
|
||||||
@@ -237,14 +542,17 @@ run_gitleaks() {
|
|||||||
--report-format json \
|
--report-format json \
|
||||||
--report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \
|
--report-path "$REPORTS_DIR/gitleaks-worktree-$name.json" \
|
||||||
--no-banner \
|
--no-banner \
|
||||||
"$repo" || status=1
|
"$repo"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
else
|
else
|
||||||
|
prepare_machine_report "$REPORTS_DIR/gitleaks-$name.json" || return 2
|
||||||
gitleaks detect \
|
gitleaks detect \
|
||||||
--source "$repo" \
|
--source "$repo" \
|
||||||
--config "$ROOT/.gitleaks.toml" \
|
--config "$ROOT/.gitleaks.toml" \
|
||||||
--report-format json \
|
--report-format json \
|
||||||
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
|
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
|
||||||
--no-banner || status=1
|
--no-banner
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
return "$status"
|
return "$status"
|
||||||
@@ -255,6 +563,7 @@ run_trivy() {
|
|||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${REPOS[@]}"; do
|
||||||
local name
|
local name
|
||||||
name="$(safe_name "$repo")"
|
name="$(safe_name "$repo")"
|
||||||
|
prepare_machine_report "$REPORTS_DIR/trivy-$name.sarif" || return 2
|
||||||
trivy fs \
|
trivy fs \
|
||||||
--scanners vuln,secret,misconfig \
|
--scanners vuln,secret,misconfig \
|
||||||
--skip-dirs node_modules \
|
--skip-dirs node_modules \
|
||||||
@@ -262,8 +571,10 @@ run_trivy() {
|
|||||||
--skip-dirs dist \
|
--skip-dirs dist \
|
||||||
--skip-dirs build \
|
--skip-dirs build \
|
||||||
--format sarif \
|
--format sarif \
|
||||||
|
--exit-code 1 \
|
||||||
--output "$REPORTS_DIR/trivy-$name.sarif" \
|
--output "$REPORTS_DIR/trivy-$name.sarif" \
|
||||||
"$repo" || status=1
|
"$repo"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
done
|
done
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
@@ -274,9 +585,11 @@ run_pip_audit_manifests() {
|
|||||||
[[ -f "$repo/requirements.txt" ]] || continue
|
[[ -f "$repo/requirements.txt" ]] || continue
|
||||||
local name
|
local name
|
||||||
name="$(safe_name "$repo")"
|
name="$(safe_name "$repo")"
|
||||||
|
prepare_machine_report "$REPORTS_DIR/pip-audit-$name.json" || return 2
|
||||||
# Requirements may contain project-relative entries such as `.[server]`.
|
# Requirements may contain project-relative entries such as `.[server]`.
|
||||||
# Resolve them from the owning repository instead of the meta-repository.
|
# Resolve them from the owning repository instead of the meta-repository.
|
||||||
(cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json") || status=1
|
(cd "$repo" && pip-audit -r requirements.txt --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json")
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
done
|
done
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
@@ -287,7 +600,9 @@ run_npm_audit_manifests() {
|
|||||||
[[ -f "$repo/webui/package-lock.json" ]] || continue
|
[[ -f "$repo/webui/package-lock.json" ]] || continue
|
||||||
local name
|
local name
|
||||||
name="$(safe_name "$repo")"
|
name="$(safe_name "$repo")"
|
||||||
(cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") || status=1
|
prepare_machine_report "$REPORTS_DIR/npm-audit-$name.json" || return 2
|
||||||
|
(cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json")
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
done
|
done
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
@@ -298,19 +613,24 @@ run_osv_scanner() {
|
|||||||
local name
|
local name
|
||||||
name="$(safe_name "$repo")"
|
name="$(safe_name "$repo")"
|
||||||
if osv-scanner scan --help >/dev/null 2>&1; then
|
if osv-scanner scan --help >/dev/null 2>&1; then
|
||||||
|
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
||||||
osv-scanner scan -r \
|
osv-scanner scan -r \
|
||||||
--allow-no-lockfiles \
|
--allow-no-lockfiles \
|
||||||
--format json \
|
--format json \
|
||||||
--output-file "$REPORTS_DIR/osv-scanner-$name.json" \
|
--output-file "$REPORTS_DIR/osv-scanner-$name.json" \
|
||||||
"$repo" || status=1
|
"$repo"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
else
|
else
|
||||||
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" || status=1
|
prepare_machine_report "$REPORTS_DIR/osv-scanner-$name.json" || return 2
|
||||||
|
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo"
|
||||||
|
accumulate_exit_status status "$?"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
|
|
||||||
run_jscpd() {
|
run_jscpd() {
|
||||||
|
prepare_machine_report "$REPORTS_DIR/jscpd/jscpd-report.json" || return 2
|
||||||
local ignored_paths
|
local ignored_paths
|
||||||
ignored_paths=(
|
ignored_paths=(
|
||||||
"**/.git/**"
|
"**/.git/**"
|
||||||
@@ -349,55 +669,66 @@ run_jscpd() {
|
|||||||
|
|
||||||
run_radon() {
|
run_radon() {
|
||||||
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
||||||
|
register_report "$REPORTS_DIR/radon-cc.txt" || return 2
|
||||||
radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt"
|
radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt"
|
||||||
}
|
}
|
||||||
|
|
||||||
run_xenon() {
|
run_xenon() {
|
||||||
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
|
||||||
|
register_report "$REPORTS_DIR/xenon.txt" || return 2
|
||||||
xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1
|
xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
write_manifest
|
register_report "$REPORTS_DIR/step-status.tsv"
|
||||||
|
register_report "$REPORTS_DIR/tool-versions.txt"
|
||||||
|
register_report "$REPORTS_DIR/workspace-state-start.tsv"
|
||||||
|
register_report "$REPORTS_DIR/workspace-state-end.tsv"
|
||||||
|
: > "$REPORTS_DIR/step-status.tsv"
|
||||||
|
if ! write_workspace_state "$REPORTS_DIR/workspace-state-start.tsv"; then
|
||||||
|
echo "Could not capture the repository state before the audit." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
run_step "Record audit tool versions" execution-only write_tool_versions
|
||||||
|
|
||||||
if has_tool semgrep; then
|
if has_tool semgrep; then
|
||||||
run_step "Semgrep SAST" run_semgrep
|
run_step "Semgrep SAST" findings-exit-one run_semgrep
|
||||||
else
|
else
|
||||||
skip_or_fail_missing semgrep
|
skip_or_fail_missing semgrep
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool bandit; then
|
if has_tool bandit; then
|
||||||
run_step "Bandit Python security scan" run_bandit
|
run_step "Bandit Python security scan" findings-exit-one run_bandit
|
||||||
else
|
else
|
||||||
skip_or_fail_missing bandit
|
skip_or_fail_missing bandit
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool ruff; then
|
if has_tool ruff; then
|
||||||
run_step "Ruff flake8-bandit security rules" run_ruff_security
|
run_step "Ruff flake8-bandit security rules" findings-exit-one run_ruff_security
|
||||||
else
|
else
|
||||||
skip_or_fail_missing ruff
|
skip_or_fail_missing ruff
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool gitleaks; then
|
if has_tool gitleaks; then
|
||||||
run_step "Gitleaks secret scan" run_gitleaks
|
run_step "Gitleaks secret scan" findings-exit-one run_gitleaks
|
||||||
else
|
else
|
||||||
skip_or_fail_missing gitleaks
|
skip_or_fail_missing gitleaks
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$MODE" != "quick" ]]; then
|
if [[ "$MODE" != "quick" ]]; then
|
||||||
if has_tool trivy; then
|
if has_tool trivy; then
|
||||||
run_step "Trivy filesystem vulnerability/secret/misconfig scan" run_trivy
|
run_step "Trivy filesystem vulnerability/secret/misconfig scan" findings-exit-one run_trivy
|
||||||
else
|
else
|
||||||
skip_or_fail_missing trivy
|
skip_or_fail_missing trivy
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool pip-audit; then
|
if has_tool pip-audit; then
|
||||||
run_step "pip-audit requirements scan" run_pip_audit_manifests
|
run_step "pip-audit requirements scan" findings-exit-one run_pip_audit_manifests
|
||||||
else
|
else
|
||||||
skip_or_fail_missing pip-audit
|
skip_or_fail_missing pip-audit
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool npm; then
|
if has_tool npm; then
|
||||||
run_step "npm audit lockfile scan" run_npm_audit_manifests
|
run_step "npm audit lockfile scan" findings-exit-one run_npm_audit_manifests
|
||||||
else
|
else
|
||||||
skip_or_fail_missing npm
|
skip_or_fail_missing npm
|
||||||
fi
|
fi
|
||||||
@@ -405,37 +736,62 @@ fi
|
|||||||
|
|
||||||
if [[ "$MODE" == "full" ]]; then
|
if [[ "$MODE" == "full" ]]; then
|
||||||
if has_tool osv-scanner; then
|
if has_tool osv-scanner; then
|
||||||
run_step "OSV-Scanner recursive dependency scan" run_osv_scanner
|
run_step "OSV-Scanner recursive dependency scan" findings-exit-one run_osv_scanner
|
||||||
else
|
else
|
||||||
skip_or_fail_missing osv-scanner
|
skip_or_fail_missing osv-scanner
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool jscpd; then
|
if has_tool jscpd; then
|
||||||
run_step "jscpd duplicate code scan" run_jscpd
|
run_step "jscpd duplicate code scan" execution-only run_jscpd
|
||||||
else
|
else
|
||||||
skip_or_fail_missing jscpd
|
skip_or_fail_missing jscpd
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool radon; then
|
if has_tool radon; then
|
||||||
run_step "Radon complexity report" run_radon
|
run_step "Radon complexity report" execution-only run_radon
|
||||||
else
|
else
|
||||||
skip_or_fail_missing radon
|
skip_or_fail_missing radon
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if has_tool xenon; then
|
if has_tool xenon; then
|
||||||
run_step "Xenon complexity threshold scan" run_xenon
|
run_step "Xenon complexity threshold scan" findings-exit-one run_xenon
|
||||||
else
|
else
|
||||||
skip_or_fail_missing xenon
|
skip_or_fail_missing xenon
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
run_step "Validate machine-readable audit reports" execution-only validate_machine_reports
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "Security audit reports written to: $REPORTS_DIR"
|
echo "Security audit reports written to: $REPORTS_DIR"
|
||||||
if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then
|
if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then
|
||||||
echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings."
|
echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings."
|
||||||
fi
|
fi
|
||||||
if [[ "$missing_status" -ne 0 ]]; then
|
if [[ "$missing_status" -ne 0 ]]; then
|
||||||
echo "One or more required tools were missing." >&2
|
echo "One or more audit tools were missing." >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! write_workspace_state "$REPORTS_DIR/workspace-state-end.tsv"; then
|
||||||
|
workspace_unchanged=false
|
||||||
|
execution_status=1
|
||||||
|
overall_status=1
|
||||||
|
echo "Could not capture the repository state after the audit." >&2
|
||||||
|
elif cmp -s "$REPORTS_DIR/workspace-state-start.tsv" "$REPORTS_DIR/workspace-state-end.tsv"; then
|
||||||
|
workspace_unchanged=true
|
||||||
|
else
|
||||||
|
workspace_unchanged=false
|
||||||
|
overall_status=1
|
||||||
|
echo "Repository state changed during the audit; discard this mixed snapshot and rerun." >&2
|
||||||
|
fi
|
||||||
|
audit_completed_at="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||||
|
if ! write_report_checksums; then
|
||||||
|
execution_status=1
|
||||||
|
overall_status=1
|
||||||
|
echo "Could not generate report checksums." >&2
|
||||||
|
fi
|
||||||
|
if ! write_manifest; then
|
||||||
|
echo "Could not write the audit manifest." >&2
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exit "$overall_status"
|
exit "$overall_status"
|
||||||
|
|||||||
130
tools/checks/check-version-alignment.py
Normal file
130
tools/checks/check-version-alignment.py
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail when backend, manifest, frontend, lock, or release-ref versions drift."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from dataclasses import asdict
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||||
|
|
||||||
|
from govoplan_release.version_alignment import ( # noqa: E402
|
||||||
|
release_composition_issues,
|
||||||
|
repository_version_issues,
|
||||||
|
selected_repository_version_issues,
|
||||||
|
)
|
||||||
|
from govoplan_release.workspace import ( # noqa: E402
|
||||||
|
load_repository_specs,
|
||||||
|
resolve_repo_path,
|
||||||
|
resolve_workspace_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--workspace-root", type=Path, default=None)
|
||||||
|
parser.add_argument("--repo", action="append", default=[], help="Repository name to check; may be repeated.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--repo-version",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
metavar="REPO=VERSION",
|
||||||
|
help="Require a repository's aligned source metadata to equal VERSION; may be repeated.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--release-composition",
|
||||||
|
action="store_true",
|
||||||
|
help="Also compare tagged backend and WebUI references in the release composition.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--source-metadata-only",
|
||||||
|
action="store_true",
|
||||||
|
help="Skip lockfile checks for the pre-commit source gate; never use for publication.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--json", action="store_true", help="Print machine-readable output.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
workspace_root = resolve_workspace_root(args.workspace_root)
|
||||||
|
expected_versions = _parse_repo_versions(parser, args.repo_version)
|
||||||
|
selected = {*args.repo, *expected_versions}
|
||||||
|
known = {spec.name for spec in load_repository_specs()}
|
||||||
|
unknown = sorted(selected - known)
|
||||||
|
if unknown:
|
||||||
|
parser.error(f"unknown repository name(s): {', '.join(unknown)}")
|
||||||
|
|
||||||
|
checked: list[str] = []
|
||||||
|
issues = list(
|
||||||
|
selected_repository_version_issues(
|
||||||
|
repo_versions=expected_versions,
|
||||||
|
workspace=workspace_root,
|
||||||
|
include_lockfiles=not args.source_metadata_only,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for spec in load_repository_specs():
|
||||||
|
if selected and spec.name not in selected:
|
||||||
|
continue
|
||||||
|
repo_path = resolve_repo_path(spec, workspace_root)
|
||||||
|
if not repo_path.exists():
|
||||||
|
continue
|
||||||
|
checked.append(spec.name)
|
||||||
|
if spec.name in expected_versions:
|
||||||
|
continue
|
||||||
|
issues.extend(
|
||||||
|
repository_version_issues(
|
||||||
|
repo_path,
|
||||||
|
include_lockfiles=not args.source_metadata_only,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.release_composition:
|
||||||
|
issues.extend(
|
||||||
|
release_composition_issues(
|
||||||
|
META_ROOT,
|
||||||
|
core_root=workspace_root / "govoplan-core",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"workspace_root": str(workspace_root),
|
||||||
|
"repositories_checked": checked,
|
||||||
|
"release_composition_checked": args.release_composition,
|
||||||
|
"issues": [asdict(issue) for issue in issues],
|
||||||
|
}
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||||
|
elif issues:
|
||||||
|
print("Version alignment failed:", file=sys.stderr)
|
||||||
|
for issue in issues:
|
||||||
|
print(
|
||||||
|
f"- {issue.repo}: {issue.source}: {issue.actual!r}; expected {issue.expected!r} "
|
||||||
|
f"({issue.message})",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
suffix = " including release composition" if args.release_composition else ""
|
||||||
|
print(f"Version alignment passed for {len(checked)} repositories{suffix}.")
|
||||||
|
return 1 if issues else 0
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_repo_versions(parser: argparse.ArgumentParser, values: list[str]) -> dict[str, str]:
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
for value in values:
|
||||||
|
if "=" not in value:
|
||||||
|
parser.error(f"--repo-version must use REPO=VERSION: {value}")
|
||||||
|
repo, version = (item.strip() for item in value.split("=", 1))
|
||||||
|
version = version.removeprefix("v")
|
||||||
|
if not repo or not version:
|
||||||
|
parser.error(f"--repo-version must use REPO=VERSION: {value}")
|
||||||
|
if repo in result and result[repo] != version:
|
||||||
|
parser.error(f"conflicting versions requested for {repo}")
|
||||||
|
result[repo] = version
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
# Application imports intentionally follow the test environment and source-path setup below.
|
||||||
|
# ruff: noqa: E402
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -31,7 +34,6 @@ from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mod
|
|||||||
from govoplan_core.core.module_installer import (
|
from govoplan_core.core.module_installer import (
|
||||||
cancel_module_installer_request,
|
cancel_module_installer_request,
|
||||||
claim_next_module_installer_request,
|
claim_next_module_installer_request,
|
||||||
module_installer_daemon_status,
|
|
||||||
module_installer_lock_status,
|
module_installer_lock_status,
|
||||||
queue_module_installer_request,
|
queue_module_installer_request,
|
||||||
read_module_installer_run,
|
read_module_installer_run,
|
||||||
@@ -44,8 +46,6 @@ from govoplan_core.core.module_installer import (
|
|||||||
)
|
)
|
||||||
from govoplan_core.core.module_management import (
|
from govoplan_core.core.module_management import (
|
||||||
ModuleInstallPlan,
|
ModuleInstallPlan,
|
||||||
ModuleInstallPlanItem,
|
|
||||||
saved_desired_enabled_modules,
|
|
||||||
saved_module_install_plan,
|
saved_module_install_plan,
|
||||||
save_desired_enabled_modules,
|
save_desired_enabled_modules,
|
||||||
save_module_install_plan,
|
save_module_install_plan,
|
||||||
|
|||||||
380
tools/checks/release_integration.py
Normal file
380
tools/checks/release_integration.py
Normal file
@@ -0,0 +1,380 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import importlib.metadata as metadata
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from collections.abc import Iterator, Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
CORE_RELEASE_TEST_MODULES = (
|
||||||
|
"tests.test_module_system",
|
||||||
|
"tests.test_access_contracts",
|
||||||
|
"tests.test_identity_organization_contracts",
|
||||||
|
"tests.test_core_events",
|
||||||
|
"tests.test_policy_contracts",
|
||||||
|
)
|
||||||
|
|
||||||
|
# These tests intentionally compare the current source workspace with sibling
|
||||||
|
# source repositories. The release gate instead installs independently tagged
|
||||||
|
# wheels and verifies the equivalent contracts below from artifact metadata and
|
||||||
|
# public module capability factories.
|
||||||
|
SOURCE_COUPLED_CORE_TESTS: Mapping[str, str] = {
|
||||||
|
"tests.test_module_system.ModuleSystemTests.test_discovers_installed_core_and_product_module_manifests": (
|
||||||
|
"asserts the current Campaign source manifest rather than the independently tagged artifact"
|
||||||
|
),
|
||||||
|
"tests.test_module_system.ModuleSystemTests.test_module_manifest_versions_match_source_project_versions": (
|
||||||
|
"requires sibling source pyproject.toml files that wheels do not contain"
|
||||||
|
),
|
||||||
|
"tests.test_module_system.ModuleSystemTests.test_release_catalog_generator_reads_manifest_interface_contracts": (
|
||||||
|
"requires the sibling meta/source workspace and is retained by the normal source gate"
|
||||||
|
),
|
||||||
|
"tests.test_identity_organization_contracts.IdentityOrganizationContractTests."
|
||||||
|
"test_sql_identity_and_organization_directories_return_dtos": (
|
||||||
|
"directly constructs the current IDM source implementation instead of using its public capability factory"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
_RELEASE_REQUIREMENT_PATTERN = re.compile(
|
||||||
|
r"^(govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://.+?\.git@v[0-9]+\.[0-9]+\.[0-9]+$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class InstalledModuleArtifact:
|
||||||
|
package_name: str
|
||||||
|
package_version: str
|
||||||
|
manifest_id: str
|
||||||
|
manifest_version: str
|
||||||
|
|
||||||
|
|
||||||
|
def release_package_names(path: Path) -> tuple[str, ...]:
|
||||||
|
names: list[str] = []
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
match = _RELEASE_REQUIREMENT_PATTERN.match(line.strip())
|
||||||
|
if match:
|
||||||
|
names.append(match.group(1))
|
||||||
|
return tuple(dict.fromkeys(names))
|
||||||
|
|
||||||
|
|
||||||
|
def artifact_contract_issues(
|
||||||
|
artifacts: Sequence[InstalledModuleArtifact],
|
||||||
|
*,
|
||||||
|
release_packages: Sequence[str],
|
||||||
|
available_manifest_versions: Mapping[str, str],
|
||||||
|
) -> tuple[str, ...]:
|
||||||
|
issues: list[str] = []
|
||||||
|
artifacts_by_package: dict[str, list[InstalledModuleArtifact]] = {}
|
||||||
|
artifacts_by_module: dict[str, list[InstalledModuleArtifact]] = {}
|
||||||
|
for artifact in artifacts:
|
||||||
|
artifacts_by_package.setdefault(artifact.package_name, []).append(artifact)
|
||||||
|
artifacts_by_module.setdefault(artifact.manifest_id, []).append(artifact)
|
||||||
|
if artifact.package_version != artifact.manifest_version:
|
||||||
|
issues.append(
|
||||||
|
f"{artifact.package_name} metadata version {artifact.package_version!r} does not match "
|
||||||
|
f"manifest {artifact.manifest_id!r} version {artifact.manifest_version!r}"
|
||||||
|
)
|
||||||
|
available_version = available_manifest_versions.get(artifact.manifest_id)
|
||||||
|
if available_version is None:
|
||||||
|
issues.append(f"Installed manifest {artifact.manifest_id!r} is not discoverable through Core")
|
||||||
|
elif available_version != artifact.manifest_version:
|
||||||
|
issues.append(
|
||||||
|
f"Discovered manifest {artifact.manifest_id!r} version {available_version!r} does not match "
|
||||||
|
f"its {artifact.package_name} entry point version {artifact.manifest_version!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for package_name in release_packages:
|
||||||
|
if package_name not in artifacts_by_package:
|
||||||
|
issues.append(f"{package_name} does not publish a govoplan.modules entry point")
|
||||||
|
for module_id, providers in artifacts_by_module.items():
|
||||||
|
if len(providers) > 1:
|
||||||
|
packages = ", ".join(sorted(item.package_name for item in providers))
|
||||||
|
issues.append(f"Module id {module_id!r} is published by multiple release packages: {packages}")
|
||||||
|
return tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def iter_test_cases(suite: unittest.TestSuite) -> Iterator[unittest.TestCase]:
|
||||||
|
for test in suite:
|
||||||
|
if isinstance(test, unittest.TestSuite):
|
||||||
|
yield from iter_test_cases(test)
|
||||||
|
else:
|
||||||
|
yield test
|
||||||
|
|
||||||
|
|
||||||
|
def filter_test_suite(
|
||||||
|
suite: unittest.TestSuite,
|
||||||
|
excluded_ids: Sequence[str],
|
||||||
|
) -> tuple[unittest.TestSuite, tuple[str, ...]]:
|
||||||
|
excluded = set(excluded_ids)
|
||||||
|
kept: list[unittest.TestCase] = []
|
||||||
|
removed: list[str] = []
|
||||||
|
for test in iter_test_cases(suite):
|
||||||
|
test_id = test.id()
|
||||||
|
if test_id in excluded:
|
||||||
|
removed.append(test_id)
|
||||||
|
else:
|
||||||
|
kept.append(test)
|
||||||
|
return unittest.TestSuite(kept), tuple(removed)
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_installed_artifacts(
|
||||||
|
package_names: Sequence[str],
|
||||||
|
) -> tuple[tuple[InstalledModuleArtifact, ...], tuple[str, ...]]:
|
||||||
|
artifacts: list[InstalledModuleArtifact] = []
|
||||||
|
issues: list[str] = []
|
||||||
|
for package_name in package_names:
|
||||||
|
try:
|
||||||
|
distribution = metadata.distribution(package_name)
|
||||||
|
except metadata.PackageNotFoundError:
|
||||||
|
issues.append(f"Release package is not installed: {package_name}")
|
||||||
|
continue
|
||||||
|
entry_points = tuple(
|
||||||
|
entry_point
|
||||||
|
for entry_point in distribution.entry_points
|
||||||
|
if entry_point.group == "govoplan.modules"
|
||||||
|
)
|
||||||
|
if not entry_points:
|
||||||
|
continue
|
||||||
|
for entry_point in entry_points:
|
||||||
|
try:
|
||||||
|
factory = entry_point.load()
|
||||||
|
manifest = factory()
|
||||||
|
manifest_id = str(manifest.id)
|
||||||
|
manifest_version = str(manifest.version)
|
||||||
|
except Exception as exc:
|
||||||
|
issues.append(f"Could not load {package_name} module entry point {entry_point.name!r}: {exc}")
|
||||||
|
continue
|
||||||
|
artifacts.append(
|
||||||
|
InstalledModuleArtifact(
|
||||||
|
package_name=package_name,
|
||||||
|
package_version=distribution.version,
|
||||||
|
manifest_id=manifest_id,
|
||||||
|
manifest_version=manifest_version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(artifacts), tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_equal(actual: object, expected: object, message: str) -> None:
|
||||||
|
if actual != expected:
|
||||||
|
raise RuntimeError(f"{message}: expected {expected!r}, got {actual!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_directory_capability_smoke(registry: object) -> None:
|
||||||
|
from govoplan_core.core.identity import CAPABILITY_IDENTITY_DIRECTORY, IdentityDirectory
|
||||||
|
from govoplan_core.core.idm import CAPABILITY_IDM_DIRECTORY, IdmDirectory
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.organizations import CAPABILITY_ORGANIZATION_DIRECTORY, OrganizationDirectory
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.db.session import configure_database, reset_database, set_database
|
||||||
|
from govoplan_core.tenancy.scope import create_scope_tables
|
||||||
|
from govoplan_identity.backend.db.models import Identity, IdentityAccountLink
|
||||||
|
from govoplan_idm.backend.db.models import IdmOrganizationFunctionAssignment
|
||||||
|
from govoplan_organizations.backend.db.models import (
|
||||||
|
OrganizationFunction,
|
||||||
|
OrganizationFunctionType,
|
||||||
|
OrganizationUnit,
|
||||||
|
OrganizationUnitType,
|
||||||
|
)
|
||||||
|
from govoplan_tenancy.backend.db.models import Tenant
|
||||||
|
|
||||||
|
registry.configure_capability_context(ModuleContext(registry=registry, settings=object()))
|
||||||
|
identity_directory = registry.require_capability(CAPABILITY_IDENTITY_DIRECTORY)
|
||||||
|
organization_directory = registry.require_capability(CAPABILITY_ORGANIZATION_DIRECTORY)
|
||||||
|
idm_directory = registry.require_capability(CAPABILITY_IDM_DIRECTORY)
|
||||||
|
if not isinstance(identity_directory, IdentityDirectory):
|
||||||
|
raise RuntimeError("Installed Identity capability does not satisfy IdentityDirectory")
|
||||||
|
if not isinstance(organization_directory, OrganizationDirectory):
|
||||||
|
raise RuntimeError("Installed Organizations capability does not satisfy OrganizationDirectory")
|
||||||
|
if not isinstance(idm_directory, IdmDirectory):
|
||||||
|
raise RuntimeError("Installed IDM capability does not satisfy IdmDirectory")
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-release-directory-") as temp_dir:
|
||||||
|
database = configure_database(
|
||||||
|
f"sqlite:///{Path(temp_dir) / 'directory.sqlite3'}",
|
||||||
|
dispose_previous=True,
|
||||||
|
)
|
||||||
|
set_database(database)
|
||||||
|
try:
|
||||||
|
create_scope_tables(database.engine)
|
||||||
|
Base.metadata.create_all(bind=database.engine)
|
||||||
|
with database.session() as session:
|
||||||
|
tenant = Tenant(
|
||||||
|
id="release-tenant",
|
||||||
|
slug="release-tenant",
|
||||||
|
name="Release Tenant",
|
||||||
|
settings={},
|
||||||
|
)
|
||||||
|
identity = Identity(
|
||||||
|
id="release-identity",
|
||||||
|
display_name="Release Person",
|
||||||
|
source="local",
|
||||||
|
)
|
||||||
|
link = IdentityAccountLink(
|
||||||
|
identity_id=identity.id,
|
||||||
|
account_id="release-account",
|
||||||
|
is_primary=True,
|
||||||
|
)
|
||||||
|
unit_type = OrganizationUnitType(
|
||||||
|
id="release-unit-type",
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
slug="office",
|
||||||
|
name="Office",
|
||||||
|
)
|
||||||
|
unit = OrganizationUnit(
|
||||||
|
id="release-unit",
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
unit_type_id=unit_type.id,
|
||||||
|
slug="registry",
|
||||||
|
name="Registry",
|
||||||
|
)
|
||||||
|
function_type = OrganizationFunctionType(
|
||||||
|
id="release-function-type",
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
slug="clerk",
|
||||||
|
name="Clerk",
|
||||||
|
organization_unit_type_id=unit_type.id,
|
||||||
|
)
|
||||||
|
function = OrganizationFunction(
|
||||||
|
id="release-function",
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
function_type_id=function_type.id,
|
||||||
|
organization_unit_id=unit.id,
|
||||||
|
slug="registry-clerk",
|
||||||
|
name="Registry Clerk",
|
||||||
|
)
|
||||||
|
assignment = IdmOrganizationFunctionAssignment(
|
||||||
|
id="release-assignment",
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
identity_id=identity.id,
|
||||||
|
function_id=function.id,
|
||||||
|
organization_unit_id=unit.id,
|
||||||
|
applies_to_subunits=True,
|
||||||
|
)
|
||||||
|
session.add_all((tenant, identity, link, unit_type, unit, function_type, function, assignment))
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
identity_ref = identity_directory.identity_for_account("release-account")
|
||||||
|
_require_equal(
|
||||||
|
identity_ref.id if identity_ref is not None else None,
|
||||||
|
"release-identity",
|
||||||
|
"Identity capability account projection",
|
||||||
|
)
|
||||||
|
function_ref = organization_directory.get_function("release-function")
|
||||||
|
_require_equal(
|
||||||
|
function_ref.name if function_ref is not None else None,
|
||||||
|
"Registry Clerk",
|
||||||
|
"Organizations capability function projection",
|
||||||
|
)
|
||||||
|
identity_assignments = idm_directory.organization_function_assignments_for_identity(
|
||||||
|
"release-identity",
|
||||||
|
tenant_id="release-tenant",
|
||||||
|
)
|
||||||
|
account_assignments = idm_directory.organization_function_assignments_for_account(
|
||||||
|
"release-account",
|
||||||
|
tenant_id="release-tenant",
|
||||||
|
)
|
||||||
|
_require_equal(
|
||||||
|
tuple(item.id for item in identity_assignments),
|
||||||
|
("release-assignment",),
|
||||||
|
"IDM identity assignment projection",
|
||||||
|
)
|
||||||
|
_require_equal(
|
||||||
|
tuple(item.id for item in account_assignments),
|
||||||
|
("release-assignment",),
|
||||||
|
"IDM account assignment projection",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
reset_database(dispose=True)
|
||||||
|
|
||||||
|
|
||||||
|
def run_installed_artifact_checks(requirements_path: Path) -> int:
|
||||||
|
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||||
|
|
||||||
|
package_names = release_package_names(requirements_path)
|
||||||
|
if not package_names:
|
||||||
|
print(f"No tagged GovOPlaN packages found in {requirements_path}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
artifacts, collection_issues = _collect_installed_artifacts(package_names)
|
||||||
|
available = available_module_manifests()
|
||||||
|
contract_issues = artifact_contract_issues(
|
||||||
|
artifacts,
|
||||||
|
release_packages=package_names,
|
||||||
|
available_manifest_versions={module_id: str(manifest.version) for module_id, manifest in available.items()},
|
||||||
|
)
|
||||||
|
issues = (*collection_issues, *contract_issues)
|
||||||
|
if issues:
|
||||||
|
print("\n".join(issues), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
module_ids = tuple(dict.fromkeys(artifact.manifest_id for artifact in artifacts))
|
||||||
|
registry = build_platform_registry(module_ids)
|
||||||
|
snapshot = registry.validate()
|
||||||
|
discovered_ids = {manifest.id for manifest in snapshot.manifests}
|
||||||
|
if discovered_ids != set(module_ids):
|
||||||
|
print(
|
||||||
|
"Release registry module mismatch: "
|
||||||
|
f"expected {', '.join(sorted(module_ids))}; got {', '.join(sorted(discovered_ids))}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 1
|
||||||
|
_run_directory_capability_smoke(registry)
|
||||||
|
versions = ", ".join(
|
||||||
|
f"{artifact.manifest_id}={artifact.manifest_version}"
|
||||||
|
for artifact in sorted(artifacts, key=lambda item: item.manifest_id)
|
||||||
|
)
|
||||||
|
print(f"Installed release artifact contracts passed: {versions}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def run_core_release_tests(core_root: Path) -> int:
|
||||||
|
resolved_root = core_root.resolve()
|
||||||
|
os.chdir(resolved_root)
|
||||||
|
sys.path.insert(0, str(resolved_root))
|
||||||
|
loader = unittest.defaultTestLoader
|
||||||
|
loaded = loader.loadTestsFromNames(CORE_RELEASE_TEST_MODULES)
|
||||||
|
suite, removed = filter_test_suite(loaded, tuple(SOURCE_COUPLED_CORE_TESTS))
|
||||||
|
missing_exclusions = set(SOURCE_COUPLED_CORE_TESTS) - set(removed)
|
||||||
|
if missing_exclusions:
|
||||||
|
print(
|
||||||
|
"Release source-test exclusions no longer match the Core suite: "
|
||||||
|
+ ", ".join(sorted(missing_exclusions)),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 1
|
||||||
|
for test_id in removed:
|
||||||
|
print(f"Artifact contract replaces source-coupled test: {test_id} ({SOURCE_COUPLED_CORE_TESTS[test_id]})")
|
||||||
|
result = unittest.TextTestRunner(verbosity=1).run(suite)
|
||||||
|
return 0 if result.wasSuccessful() else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _parser() -> argparse.ArgumentParser:
|
||||||
|
meta_root = Path(__file__).resolve().parents[2]
|
||||||
|
parser = argparse.ArgumentParser(description="Artifact-aware GovOPlaN release integration checks")
|
||||||
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
artifact_parser = subparsers.add_parser("artifacts", help="Validate installed module artifacts and capabilities")
|
||||||
|
artifact_parser.add_argument(
|
||||||
|
"--requirements",
|
||||||
|
type=Path,
|
||||||
|
default=meta_root / "requirements-release.txt",
|
||||||
|
)
|
||||||
|
core_parser = subparsers.add_parser("core-tests", help="Run Core tests that are valid for tagged module artifacts")
|
||||||
|
core_parser.add_argument("--core-root", type=Path, required=True)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
args = _parser().parse_args(argv)
|
||||||
|
if args.command == "artifacts":
|
||||||
|
return run_installed_artifact_checks(args.requirements)
|
||||||
|
return run_core_release_tests(args.core_root)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -31,19 +31,24 @@ RUN apt-get update \
|
|||||||
COPY --from=go-tools /go/bin/gitleaks /usr/local/bin/gitleaks
|
COPY --from=go-tools /go/bin/gitleaks /usr/local/bin/gitleaks
|
||||||
COPY --from=go-tools /go/bin/osv-scanner /usr/local/bin/osv-scanner
|
COPY --from=go-tools /go/bin/osv-scanner /usr/local/bin/osv-scanner
|
||||||
COPY requirements-audit.txt /tmp/requirements-audit.txt
|
COPY requirements-audit.txt /tmp/requirements-audit.txt
|
||||||
|
COPY tools/checks/security-audit/semgrep-govoplan.yml /tmp/semgrep-govoplan.yml
|
||||||
|
COPY tools/checks/check-version-alignment.py /tmp/semgrep-smoke.py
|
||||||
|
|
||||||
RUN python -m pip install --upgrade pip \
|
RUN python -m pip install --upgrade pip \
|
||||||
&& python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \
|
&& python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \
|
||||||
&& python -m pip install --no-cache-dir "semgrep>=1.140,<2" \
|
&& python -m pip install --no-cache-dir "semgrep>=1.140,<2" \
|
||||||
&& python -m pip install --no-cache-dir --upgrade --no-deps "click>=8.3.3" \
|
&& python -m pip install --no-cache-dir --upgrade --no-deps "click==8.3.3" \
|
||||||
|
&& python -m pip install --no-cache-dir --upgrade --no-deps "mcp==1.28.1" \
|
||||||
&& npm install -g jscpd \
|
&& npm install -g jscpd \
|
||||||
&& semgrep --version \
|
&& semgrep --version \
|
||||||
|
&& semgrep scan --metrics=off --config /tmp/semgrep-govoplan.yml /tmp/semgrep-smoke.py \
|
||||||
&& bandit --version \
|
&& bandit --version \
|
||||||
&& ruff --version \
|
&& ruff --version \
|
||||||
&& gitleaks version \
|
&& gitleaks version \
|
||||||
&& osv-scanner --version \
|
&& osv-scanner --version \
|
||||||
&& trivy --version \
|
&& trivy --version \
|
||||||
&& jscpd --version
|
&& jscpd --version \
|
||||||
|
&& pip-audit --progress-spinner off
|
||||||
|
|
||||||
RUN mkdir -p /workspace \
|
RUN mkdir -p /workspace \
|
||||||
&& chmod 0777 /workspace
|
&& chmod 0777 /workspace
|
||||||
|
|||||||
@@ -84,6 +84,8 @@ IMAGE_FINGERPRINT="$(
|
|||||||
{
|
{
|
||||||
sha256sum "$ROOT/tools/checks/security-audit/Dockerfile"
|
sha256sum "$ROOT/tools/checks/security-audit/Dockerfile"
|
||||||
sha256sum "$ROOT/requirements-audit.txt"
|
sha256sum "$ROOT/requirements-audit.txt"
|
||||||
|
sha256sum "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml"
|
||||||
|
sha256sum "$ROOT/tools/checks/check-version-alignment.py"
|
||||||
} | sha256sum | cut -c1-16
|
} | sha256sum | cut -c1-16
|
||||||
)"
|
)"
|
||||||
FINGERPRINT_IMAGE="${SECURITY_AUDIT_FINGERPRINT_IMAGE:-$IMAGE_REPOSITORY:$IMAGE_FINGERPRINT}"
|
FINGERPRINT_IMAGE="${SECURITY_AUDIT_FINGERPRINT_IMAGE:-$IMAGE_REPOSITORY:$IMAGE_FINGERPRINT}"
|
||||||
@@ -129,6 +131,7 @@ docker run --rm \
|
|||||||
-e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \
|
-e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \
|
||||||
-e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \
|
-e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \
|
||||||
-e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \
|
-e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \
|
||||||
|
-e SECURITY_AUDIT_TOOLBOX_FINGERPRINT="$IMAGE_FINGERPRINT" \
|
||||||
"${EXTRA_ENV[@]}" \
|
"${EXTRA_ENV[@]}" \
|
||||||
-v "$MOUNT_ROOT:/workspace" \
|
-v "$MOUNT_ROOT:/workspace" \
|
||||||
-w "$WORKDIR" \
|
-w "$WORKDIR" \
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import argparse
|
|||||||
import base64
|
import base64
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
|||||||
@@ -19,9 +19,11 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|||||||
META_ROOT = Path(__file__).resolve().parents[2]
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
|
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
|
||||||
sys.path.insert(0, str(CORE_ROOT / "src"))
|
sys.path.insert(0, str(CORE_ROOT / "src"))
|
||||||
|
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||||
|
|
||||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||||
|
from govoplan_release.version_alignment import selected_repository_version_issues # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||||
@@ -196,6 +198,19 @@ def main() -> int:
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
version = args.version.removeprefix("v")
|
version = args.version.removeprefix("v")
|
||||||
|
version_issues = selected_repository_version_issues(
|
||||||
|
repo_versions={
|
||||||
|
"govoplan-core": version,
|
||||||
|
**{module.repo: version for module in CATALOG_MODULES},
|
||||||
|
},
|
||||||
|
workspace=CORE_ROOT.parent,
|
||||||
|
)
|
||||||
|
if version_issues:
|
||||||
|
details = "; ".join(
|
||||||
|
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
|
||||||
|
for issue in version_issues
|
||||||
|
)
|
||||||
|
parser.error(f"version alignment gate failed: {details}")
|
||||||
tag = f"v{version}"
|
tag = f"v{version}"
|
||||||
generated_at = datetime.now(tz=UTC)
|
generated_at = datetime.now(tz=UTC)
|
||||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||||
|
|||||||
@@ -17,6 +17,10 @@ the regenerated release lockfile.
|
|||||||
Options:
|
Options:
|
||||||
--npm <path> npm executable to use.
|
--npm <path> npm executable to use.
|
||||||
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
|
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
|
||||||
|
--local-git-repo <path>
|
||||||
|
Resolve that repository's release tag from its local Git
|
||||||
|
object store while preserving the published Git URL in
|
||||||
|
the lockfile. May be repeated.
|
||||||
-h, --help Show this help.
|
-h, --help Show this help.
|
||||||
USAGE
|
USAGE
|
||||||
}
|
}
|
||||||
@@ -32,6 +36,7 @@ CORE_ROOT="$(cd "$CORE_ROOT" && pwd)"
|
|||||||
WEBUI="$CORE_ROOT/webui"
|
WEBUI="$CORE_ROOT/webui"
|
||||||
NPM_BIN="${NPM:-}"
|
NPM_BIN="${NPM:-}"
|
||||||
NODE_BIN="${NODE:-}"
|
NODE_BIN="${NODE:-}"
|
||||||
|
LOCAL_GIT_REPOS=()
|
||||||
|
|
||||||
if [[ -z "$NPM_BIN" ]]; then
|
if [[ -z "$NPM_BIN" ]]; then
|
||||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||||
@@ -61,6 +66,11 @@ while [[ $# -gt 0 ]]; do
|
|||||||
WEBUI="$CORE_ROOT/webui"
|
WEBUI="$CORE_ROOT/webui"
|
||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
|
--local-git-repo)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
LOCAL_GIT_REPOS+=("$(cd "$2" && pwd)")
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
-h|--help)
|
-h|--help)
|
||||||
usage
|
usage
|
||||||
exit 0
|
exit 0
|
||||||
@@ -90,9 +100,27 @@ fi
|
|||||||
|
|
||||||
echo "Generating release lockfile from $WEBUI/package.release.json"
|
echo "Generating release lockfile from $WEBUI/package.release.json"
|
||||||
echo "Temporary workspace: $TMP_DIR"
|
echo "Temporary workspace: $TMP_DIR"
|
||||||
|
|
||||||
|
GIT_ENV=(env)
|
||||||
|
git_config_count=0
|
||||||
|
for local_repo in "${LOCAL_GIT_REPOS[@]}"; do
|
||||||
|
[[ -d "$local_repo/.git" ]] || fail "local Git release source is not a repository: $local_repo"
|
||||||
|
remote_url="$(git -C "$local_repo" remote get-url origin)"
|
||||||
|
remote_url="${remote_url#git+}"
|
||||||
|
if [[ "$remote_url" != *://* && "$remote_url" =~ ^([^@]+@)?([^:]+):(.+)$ ]]; then
|
||||||
|
remote_url="ssh://${BASH_REMATCH[1]}${BASH_REMATCH[2]}/${BASH_REMATCH[3]}"
|
||||||
|
fi
|
||||||
|
GIT_ENV+=(
|
||||||
|
"GIT_CONFIG_KEY_${git_config_count}=url.file://$local_repo/.insteadOf"
|
||||||
|
"GIT_CONFIG_VALUE_${git_config_count}=$remote_url"
|
||||||
|
)
|
||||||
|
git_config_count=$((git_config_count + 1))
|
||||||
|
done
|
||||||
|
GIT_ENV+=("GIT_CONFIG_COUNT=$git_config_count")
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$TMP_DIR"
|
cd "$TMP_DIR"
|
||||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
"${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||||
mapfile -t GIT_PACKAGES < <(
|
mapfile -t GIT_PACKAGES < <(
|
||||||
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
|
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
@@ -108,7 +136,7 @@ NODE
|
|||||||
)
|
)
|
||||||
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
|
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
|
||||||
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
|
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
|
||||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
"${GIT_ENV[@]}" PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
||||||
fi
|
fi
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
112
tools/release/generate-release-sbom.py
Normal file
112
tools/release/generate-release-sbom.py
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate a CycloneDX SBOM from the resolved GovOPlaN release environment."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
|
||||||
|
META_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(META_ROOT / "tools" / "release"))
|
||||||
|
|
||||||
|
from govoplan_release.sbom import ( # noqa: E402
|
||||||
|
build_release_sbom,
|
||||||
|
parse_sbom_timestamp,
|
||||||
|
render_sbom,
|
||||||
|
sbom_sha256,
|
||||||
|
timestamp_from_source_date_epoch,
|
||||||
|
validate_sbom,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--python", default=sys.executable, help="Python executable for pip inspect.")
|
||||||
|
parser.add_argument("--core-root", type=Path, default=META_ROOT.parent / "govoplan-core")
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
default=META_ROOT / "runtime" / "release-artifacts" / "govoplan-sbom.cdx.json",
|
||||||
|
)
|
||||||
|
parser.add_argument("--pip-inspect", type=Path, help="Use an existing pip inspect report instead of executing pip.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--timestamp",
|
||||||
|
help="ISO-8601 build timestamp. Defaults to SOURCE_DATE_EPOCH, then the current time.",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
core_root = args.core_root.expanduser().resolve()
|
||||||
|
product_version = _project_version(core_root / "pyproject.toml")
|
||||||
|
npm_lock_path = core_root / "webui" / "package-lock.release.json"
|
||||||
|
if not npm_lock_path.exists():
|
||||||
|
parser.error(f"release npm lockfile does not exist: {npm_lock_path}")
|
||||||
|
|
||||||
|
pip_inspect = _read_json(args.pip_inspect) if args.pip_inspect else _run_pip_inspect(args.python)
|
||||||
|
npm_lock = _read_json(npm_lock_path)
|
||||||
|
try:
|
||||||
|
timestamp = _resolved_timestamp(args.timestamp)
|
||||||
|
payload = build_release_sbom(
|
||||||
|
product_version=product_version,
|
||||||
|
pip_inspect=pip_inspect,
|
||||||
|
npm_lock=npm_lock,
|
||||||
|
timestamp=timestamp,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
parser.error(str(exc))
|
||||||
|
validate_sbom(payload)
|
||||||
|
|
||||||
|
output = args.output.expanduser().resolve()
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(render_sbom(payload), encoding="utf-8")
|
||||||
|
print(f"Wrote {len(payload['components'])} components to {output}")
|
||||||
|
print(f"SHA-256: {sbom_sha256(payload)}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _run_pip_inspect(python: str) -> dict:
|
||||||
|
result = subprocess.run(
|
||||||
|
[python, "-m", "pip", "inspect", "--local"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise SystemExit(f"pip inspect failed: {result.stderr.strip()}")
|
||||||
|
return json.loads(result.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_json(path: Path) -> dict:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError(f"expected a JSON object in {path}")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _project_version(path: Path) -> str:
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
payload = tomllib.load(handle)
|
||||||
|
project = payload.get("project")
|
||||||
|
if not isinstance(project, dict) or not isinstance(project.get("version"), str):
|
||||||
|
raise ValueError(f"project.version is missing from {path}")
|
||||||
|
return project["version"]
|
||||||
|
|
||||||
|
|
||||||
|
def _resolved_timestamp(explicit: str | None):
|
||||||
|
if explicit is not None:
|
||||||
|
return parse_sbom_timestamp(explicit)
|
||||||
|
source_date_epoch = os.environ.get("SOURCE_DATE_EPOCH")
|
||||||
|
if source_date_epoch is not None:
|
||||||
|
return timestamp_from_source_date_epoch(source_date_epoch)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -99,6 +99,7 @@ def collect_versions(path: Path) -> VersionSnapshot:
|
|||||||
package=read_json_version(path / "package.json"),
|
package=read_json_version(path / "package.json"),
|
||||||
webui_package=read_json_version(path / "webui" / "package.json"),
|
webui_package=read_json_version(path / "webui" / "package.json"),
|
||||||
manifests=read_manifest_versions(path),
|
manifests=read_manifest_versions(path),
|
||||||
|
package_inits=read_package_init_versions(path),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -137,6 +138,21 @@ def read_manifest_versions(path: Path) -> tuple[str, ...]:
|
|||||||
return tuple(versions)
|
return tuple(versions)
|
||||||
|
|
||||||
|
|
||||||
|
def read_package_init_versions(path: Path) -> tuple[str, ...]:
|
||||||
|
"""Read public runtime versions declared by top-level Python packages."""
|
||||||
|
|
||||||
|
src = path / "src"
|
||||||
|
if not src.exists():
|
||||||
|
return ()
|
||||||
|
versions: list[str] = []
|
||||||
|
for package_init in sorted(src.glob("*/__init__.py")):
|
||||||
|
text = package_init.read_text(encoding="utf-8")
|
||||||
|
match = re.search(r'(?m)^__version__\s*=\s*["\']([^"\']+)["\']', text)
|
||||||
|
if match is not None:
|
||||||
|
versions.append(match.group(1))
|
||||||
|
return tuple(versions)
|
||||||
|
|
||||||
|
|
||||||
def git_text(path: Path, *args: str, timeout: int = 10) -> str:
|
def git_text(path: Path, *args: str, timeout: int = 10) -> str:
|
||||||
result = git(path, *args, timeout=timeout)
|
result = git(path, *args, timeout=timeout)
|
||||||
return result.stdout.strip() if result.returncode == 0 else ""
|
return result.stdout.strip() if result.returncode == 0 else ""
|
||||||
|
|||||||
@@ -26,10 +26,17 @@ class VersionSnapshot:
|
|||||||
package: str | None = None
|
package: str | None = None
|
||||||
webui_package: str | None = None
|
webui_package: str | None = None
|
||||||
manifests: tuple[str, ...] = ()
|
manifests: tuple[str, ...] = ()
|
||||||
|
package_inits: tuple[str, ...] = ()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def primary(self) -> str | None:
|
def primary(self) -> str | None:
|
||||||
return self.pyproject or self.package or self.webui_package or (self.manifests[0] if self.manifests else None)
|
return (
|
||||||
|
self.pyproject
|
||||||
|
or self.package
|
||||||
|
or self.webui_package
|
||||||
|
or (self.manifests[0] if self.manifests else None)
|
||||||
|
or (self.package_inits[0] if self.package_inits else None)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ from govoplan_core.core.module_package_catalog import validate_module_package_ca
|
|||||||
|
|
||||||
from .catalog import canonical_hash
|
from .catalog import canonical_hash
|
||||||
from .model import CatalogPublishResult, CatalogPublishStep
|
from .model import CatalogPublishResult, CatalogPublishStep
|
||||||
|
from .module_directory import write_module_directory
|
||||||
from .selective_catalog import trusted_keys_from_keyring
|
from .selective_catalog import trusted_keys_from_keyring
|
||||||
|
from .version_alignment import candidate_catalog_version_issues
|
||||||
from .workspace import DEFAULT_WORKSPACE_ROOT, resolve_workspace_root, website_root
|
from .workspace import DEFAULT_WORKSPACE_ROOT, resolve_workspace_root, website_root
|
||||||
|
|
||||||
|
|
||||||
@@ -41,7 +43,6 @@ def publish_catalog_candidate(
|
|||||||
resolved_web_root = Path(web_root).expanduser().resolve() if web_root is not None else website_root(workspace)
|
resolved_web_root = Path(web_root).expanduser().resolve() if web_root is not None else website_root(workspace)
|
||||||
candidate_catalog = candidate_root / "channels" / f"{channel}.json"
|
candidate_catalog = candidate_root / "channels" / f"{channel}.json"
|
||||||
candidate_keyring = candidate_root / "keyring.json"
|
candidate_keyring = candidate_root / "keyring.json"
|
||||||
candidate_modules = candidate_root / "modules"
|
|
||||||
target_catalog = resolved_web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json"
|
target_catalog = resolved_web_root / "public" / "catalogs" / "v1" / "channels" / f"{channel}.json"
|
||||||
target_keyring = resolved_web_root / "public" / "catalogs" / "v1" / "keyring.json"
|
target_keyring = resolved_web_root / "public" / "catalogs" / "v1" / "keyring.json"
|
||||||
target_modules = resolved_web_root / "public" / "catalogs" / "v1" / "modules"
|
target_modules = resolved_web_root / "public" / "catalogs" / "v1" / "modules"
|
||||||
@@ -72,11 +73,36 @@ def publish_catalog_candidate(
|
|||||||
validation_warnings: tuple[str, ...] = ()
|
validation_warnings: tuple[str, ...] = ()
|
||||||
|
|
||||||
if candidate_payload is not None and keyring_payload is not None:
|
if candidate_payload is not None and keyring_payload is not None:
|
||||||
|
version_issues = candidate_catalog_version_issues(candidate_payload)
|
||||||
|
blockers.extend(
|
||||||
|
f"candidate version alignment: {issue.source}: {issue.actual!r}; "
|
||||||
|
f"expected {issue.expected!r} ({issue.message})"
|
||||||
|
for issue in version_issues
|
||||||
|
)
|
||||||
|
candidate_keys = trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {})
|
||||||
|
target_keyring_payload = read_json(target_keyring) if target_keyring.exists() else None
|
||||||
|
publication_trust = trusted_keys_from_keyring(
|
||||||
|
target_keyring_payload if isinstance(target_keyring_payload, dict) else {}
|
||||||
|
)
|
||||||
|
if not publication_trust:
|
||||||
|
blockers.append(
|
||||||
|
"publication trust anchor is missing; bootstrap a trusted website keyring through a separate reviewed process"
|
||||||
|
)
|
||||||
|
for key_id in sorted(set(candidate_keys) & set(publication_trust)):
|
||||||
|
if candidate_keys[key_id] != publication_trust[key_id]:
|
||||||
|
blockers.append(f"candidate keyring changes the public key for trusted key id {key_id!r}")
|
||||||
|
if candidate_keyring_hash != target_keyring_hash_before:
|
||||||
|
release = candidate_payload.get("release") if isinstance(candidate_payload, dict) else None
|
||||||
|
signed_keyring_hash = release.get("keyring_sha256") if isinstance(release, dict) else None
|
||||||
|
if signed_keyring_hash != candidate_keyring_hash:
|
||||||
|
blockers.append(
|
||||||
|
"candidate keyring differs from the publication trust anchor without a matching signed keyring_sha256"
|
||||||
|
)
|
||||||
validation = validate_module_package_catalog(
|
validation = validate_module_package_catalog(
|
||||||
candidate_catalog,
|
candidate_catalog,
|
||||||
require_trusted=True,
|
require_trusted=True,
|
||||||
approved_channels=(channel,),
|
approved_channels=(channel,),
|
||||||
trusted_keys=trusted_keys_from_keyring(keyring_payload if isinstance(keyring_payload, dict) else {}),
|
trusted_keys=publication_trust,
|
||||||
)
|
)
|
||||||
validation_valid = validation.get("valid") is True
|
validation_valid = validation.get("valid") is True
|
||||||
validation_error = str(validation["error"]) if validation.get("error") else None
|
validation_error = str(validation["error"]) if validation.get("error") else None
|
||||||
@@ -115,14 +141,12 @@ def publish_catalog_candidate(
|
|||||||
steps.append(
|
steps.append(
|
||||||
CatalogPublishStep(
|
CatalogPublishStep(
|
||||||
id="copy",
|
id="copy",
|
||||||
title="Copy candidate catalog, keyring, and module directory",
|
title="Copy candidate catalog/keyring and derive the module directory",
|
||||||
detail=f"{candidate_root} -> {resolved_web_root / 'public' / 'catalogs' / 'v1'}",
|
detail=f"{candidate_root} -> {resolved_web_root / 'public' / 'catalogs' / 'v1'}",
|
||||||
mutating=True,
|
mutating=True,
|
||||||
status="planned" if not apply else "blocked" if blockers else "done",
|
status="planned" if not apply else "blocked" if blockers else "done",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not candidate_modules.exists():
|
|
||||||
notes.append("Candidate has no module-directory tree; only catalog and keyring will be copied.")
|
|
||||||
if build_web:
|
if build_web:
|
||||||
steps.append(
|
steps.append(
|
||||||
CatalogPublishStep(
|
CatalogPublishStep(
|
||||||
@@ -224,10 +248,15 @@ def publish_catalog_candidate(
|
|||||||
target_catalog.parent.mkdir(parents=True, exist_ok=True)
|
target_catalog.parent.mkdir(parents=True, exist_ok=True)
|
||||||
shutil.copy2(candidate_catalog, target_catalog)
|
shutil.copy2(candidate_catalog, target_catalog)
|
||||||
shutil.copy2(candidate_keyring, target_keyring)
|
shutil.copy2(candidate_keyring, target_keyring)
|
||||||
if candidate_modules.exists():
|
if target_modules.exists():
|
||||||
if target_modules.exists():
|
shutil.rmtree(target_modules)
|
||||||
shutil.rmtree(target_modules)
|
write_module_directory(
|
||||||
shutil.copytree(candidate_modules, target_modules)
|
catalog_payload=candidate_payload,
|
||||||
|
keyring_payload=keyring_payload,
|
||||||
|
output_root=target_catalog.parent.parent,
|
||||||
|
channel=channel,
|
||||||
|
public_base_url=catalog_public_base_url(candidate_payload),
|
||||||
|
)
|
||||||
|
|
||||||
completed_steps = [replace_status(step, "done") if step.id == "copy" else step for step in steps]
|
completed_steps = [replace_status(step, "done") if step.id == "copy" else step for step in steps]
|
||||||
if build_web:
|
if build_web:
|
||||||
@@ -282,6 +311,14 @@ def publish_catalog_candidate(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def catalog_public_base_url(payload: dict[str, object]) -> str:
|
||||||
|
release = payload.get("release")
|
||||||
|
catalog_url = release.get("catalog_url") if isinstance(release, dict) else None
|
||||||
|
if isinstance(catalog_url, str) and "/catalogs/" in catalog_url:
|
||||||
|
return catalog_url.split("/catalogs/", 1)[0]
|
||||||
|
return "https://govoplan.add-ideas.de"
|
||||||
|
|
||||||
|
|
||||||
def result(
|
def result(
|
||||||
*,
|
*,
|
||||||
status: str,
|
status: str,
|
||||||
|
|||||||
235
tools/release/govoplan_release/sbom.py
Normal file
235
tools/release/govoplan_release/sbom.py
Normal file
@@ -0,0 +1,235 @@
|
|||||||
|
"""CycloneDX SBOM construction from resolved Python and npm environments."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import quote, urlsplit
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
def build_release_sbom(
|
||||||
|
*,
|
||||||
|
product_version: str,
|
||||||
|
pip_inspect: dict[str, Any],
|
||||||
|
npm_lock: dict[str, Any],
|
||||||
|
timestamp: datetime | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
components = _deduplicate_components(
|
||||||
|
[
|
||||||
|
*python_components(pip_inspect),
|
||||||
|
*npm_components(npm_lock),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
root_ref = f"pkg:generic/govoplan@{quote(product_version, safe='') }"
|
||||||
|
resolved_timestamp = _normalized_timestamp(timestamp)
|
||||||
|
timestamp_text = resolved_timestamp.isoformat().replace("+00:00", "Z")
|
||||||
|
serial_seed = json.dumps(
|
||||||
|
{
|
||||||
|
"product_version": product_version,
|
||||||
|
"timestamp": timestamp_text,
|
||||||
|
"components": components,
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
serial = uuid.uuid5(uuid.NAMESPACE_URL, f"https://govoplan.add-ideas.de/sbom/{serial_seed}")
|
||||||
|
return {
|
||||||
|
"$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json",
|
||||||
|
"bomFormat": "CycloneDX",
|
||||||
|
"specVersion": "1.6",
|
||||||
|
"serialNumber": f"urn:uuid:{serial}",
|
||||||
|
"version": 1,
|
||||||
|
"metadata": {
|
||||||
|
"timestamp": timestamp_text,
|
||||||
|
"lifecycles": [{"phase": "build"}],
|
||||||
|
"component": {
|
||||||
|
"type": "application",
|
||||||
|
"bom-ref": root_ref,
|
||||||
|
"group": "ADD ideas UG",
|
||||||
|
"name": "GovOPlaN",
|
||||||
|
"version": product_version,
|
||||||
|
"purl": root_ref,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"components": components,
|
||||||
|
"dependencies": [
|
||||||
|
{
|
||||||
|
"ref": root_ref,
|
||||||
|
"dependsOn": [component["bom-ref"] for component in components],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def python_components(report: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
if str(report.get("version")) != "1":
|
||||||
|
raise ValueError("unsupported pip inspect report version")
|
||||||
|
installed = report.get("installed")
|
||||||
|
if not isinstance(installed, list):
|
||||||
|
raise ValueError("pip inspect report is missing installed distributions")
|
||||||
|
|
||||||
|
components: list[dict[str, Any]] = []
|
||||||
|
for item in installed:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
metadata = item.get("metadata")
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
continue
|
||||||
|
name = metadata.get("name")
|
||||||
|
version = metadata.get("version")
|
||||||
|
if not isinstance(name, str) or not isinstance(version, str):
|
||||||
|
continue
|
||||||
|
normalized = name.lower().replace("_", "-")
|
||||||
|
purl = f"pkg:pypi/{quote(normalized, safe='')}@{quote(version, safe='')}"
|
||||||
|
component: dict[str, Any] = {
|
||||||
|
"type": "library",
|
||||||
|
"bom-ref": purl,
|
||||||
|
"name": name,
|
||||||
|
"version": version,
|
||||||
|
"purl": purl,
|
||||||
|
"scope": "required",
|
||||||
|
}
|
||||||
|
direct_url = item.get("direct_url")
|
||||||
|
if isinstance(direct_url, dict) and isinstance(direct_url.get("url"), str):
|
||||||
|
distribution_url = _public_distribution_url(direct_url["url"])
|
||||||
|
if distribution_url is not None:
|
||||||
|
component["externalReferences"] = [{"type": "distribution", "url": distribution_url}]
|
||||||
|
components.append(component)
|
||||||
|
return components
|
||||||
|
|
||||||
|
|
||||||
|
def npm_components(lock: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
packages = lock.get("packages")
|
||||||
|
if not isinstance(packages, dict):
|
||||||
|
raise ValueError("npm lockfile is missing the packages map")
|
||||||
|
|
||||||
|
components: list[dict[str, Any]] = []
|
||||||
|
for path, item in packages.items():
|
||||||
|
if not path or not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
name = item.get("name") or _npm_name_from_lock_path(str(path))
|
||||||
|
version = item.get("version")
|
||||||
|
if not isinstance(name, str) or not isinstance(version, str):
|
||||||
|
continue
|
||||||
|
purl = f"pkg:npm/{quote(name, safe='/')}@{quote(version, safe='')}"
|
||||||
|
component: dict[str, Any] = {
|
||||||
|
"type": "library",
|
||||||
|
"bom-ref": purl,
|
||||||
|
"name": name,
|
||||||
|
"version": version,
|
||||||
|
"purl": purl,
|
||||||
|
"scope": "optional" if item.get("dev") is True else "required",
|
||||||
|
}
|
||||||
|
hashes = _sri_hashes(item.get("integrity"))
|
||||||
|
if hashes:
|
||||||
|
component["hashes"] = hashes
|
||||||
|
resolved = item.get("resolved")
|
||||||
|
if isinstance(resolved, str):
|
||||||
|
distribution_url = _public_distribution_url(resolved)
|
||||||
|
if distribution_url is not None:
|
||||||
|
component["externalReferences"] = [{"type": "distribution", "url": distribution_url}]
|
||||||
|
components.append(component)
|
||||||
|
return components
|
||||||
|
|
||||||
|
|
||||||
|
def validate_sbom(payload: dict[str, Any]) -> None:
|
||||||
|
if payload.get("bomFormat") != "CycloneDX" or payload.get("specVersion") != "1.6":
|
||||||
|
raise ValueError("not a supported CycloneDX 1.6 SBOM")
|
||||||
|
metadata = payload.get("metadata")
|
||||||
|
if not isinstance(metadata, dict) or not isinstance(metadata.get("component"), dict):
|
||||||
|
raise ValueError("SBOM metadata component is missing")
|
||||||
|
components = payload.get("components")
|
||||||
|
if not isinstance(components, list) or not components:
|
||||||
|
raise ValueError("SBOM has no dependency components")
|
||||||
|
refs = [item.get("bom-ref") for item in components if isinstance(item, dict)]
|
||||||
|
if any(not isinstance(ref, str) or not ref for ref in refs):
|
||||||
|
raise ValueError("SBOM component is missing bom-ref")
|
||||||
|
if len(refs) != len(set(refs)):
|
||||||
|
raise ValueError("SBOM contains duplicate component references")
|
||||||
|
|
||||||
|
|
||||||
|
def sbom_sha256(payload: dict[str, Any]) -> str:
|
||||||
|
return hashlib.sha256(render_sbom(payload).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def render_sbom(payload: dict[str, Any]) -> str:
|
||||||
|
return json.dumps(payload, indent=2, sort_keys=True) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def timestamp_from_source_date_epoch(value: str) -> datetime:
|
||||||
|
"""Resolve the reproducible-build timestamp convention to UTC."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
epoch = int(value)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError("SOURCE_DATE_EPOCH must be an integer Unix timestamp") from exc
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(epoch, tz=UTC)
|
||||||
|
except (OverflowError, OSError) as exc:
|
||||||
|
raise ValueError("SOURCE_DATE_EPOCH is outside the supported timestamp range") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def parse_sbom_timestamp(value: str) -> datetime:
|
||||||
|
"""Parse an explicit ISO-8601 SBOM timestamp and normalize it to UTC."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError("timestamp must be an ISO-8601 value with a timezone") from exc
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
raise ValueError("timestamp must include a timezone")
|
||||||
|
return parsed.astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_timestamp(value: datetime | None) -> datetime:
|
||||||
|
resolved = value or datetime.now(tz=UTC)
|
||||||
|
if resolved.tzinfo is None:
|
||||||
|
raise ValueError("SBOM timestamp must include a timezone")
|
||||||
|
return resolved.astimezone(UTC).replace(microsecond=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _deduplicate_components(components: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
unique = {component["bom-ref"]: component for component in components}
|
||||||
|
return [unique[key] for key in sorted(unique)]
|
||||||
|
|
||||||
|
|
||||||
|
def _npm_name_from_lock_path(path: str) -> str:
|
||||||
|
marker = "node_modules/"
|
||||||
|
if marker not in path:
|
||||||
|
return Path(path).name
|
||||||
|
return path.rsplit(marker, 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def _sri_hashes(value: object) -> list[dict[str, str]]:
|
||||||
|
if not isinstance(value, str) or "-" not in value:
|
||||||
|
return []
|
||||||
|
algorithm, encoded = value.split("-", 1)
|
||||||
|
algorithm_name = {"sha256": "SHA-256", "sha384": "SHA-384", "sha512": "SHA-512"}.get(algorithm.lower())
|
||||||
|
if algorithm_name is None:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
content = base64.b64decode(encoded, validate=True).hex()
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
|
return [{"alg": algorithm_name, "content": content}]
|
||||||
|
|
||||||
|
|
||||||
|
def _public_distribution_url(value: str) -> str | None:
|
||||||
|
"""Keep stable public provenance without leaking local paths or credentials."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(value)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if parsed.scheme.lower() not in {"git", "git+https", "http", "https"}:
|
||||||
|
return None
|
||||||
|
if not parsed.hostname or parsed.username is not None or parsed.password is not None:
|
||||||
|
return None
|
||||||
|
if parsed.query:
|
||||||
|
return None
|
||||||
|
return value
|
||||||
@@ -19,6 +19,7 @@ from .contracts import collect_contracts
|
|||||||
from .git_state import collect_repository_snapshot
|
from .git_state import collect_repository_snapshot
|
||||||
from .model import CatalogEntryChange, SelectiveCatalogCandidate
|
from .model import CatalogEntryChange, SelectiveCatalogCandidate
|
||||||
from .module_directory import write_module_directory
|
from .module_directory import write_module_directory
|
||||||
|
from .version_alignment import selected_repository_version_issues
|
||||||
from .workspace import load_repository_specs, resolve_workspace_root, website_root
|
from .workspace import load_repository_specs, resolve_workspace_root, website_root
|
||||||
|
|
||||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||||
@@ -46,6 +47,7 @@ def build_selective_catalog_candidate(
|
|||||||
raise ValueError("At least one signing key is required.")
|
raise ValueError("At least one signing key is required.")
|
||||||
|
|
||||||
workspace = resolve_workspace_root(workspace_root)
|
workspace = resolve_workspace_root(workspace_root)
|
||||||
|
enforce_selected_version_alignment(repo_versions=repo_versions, workspace=workspace)
|
||||||
web_root = website_root(workspace)
|
web_root = website_root(workspace)
|
||||||
source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel)
|
source_catalog = resolve_base_catalog(base_catalog=base_catalog, web_root=web_root, channel=channel)
|
||||||
payload = read_catalog(source_catalog)
|
payload = read_catalog(source_catalog)
|
||||||
@@ -75,22 +77,24 @@ def build_selective_catalog_candidate(
|
|||||||
repository_base=repository_base.rstrip("/"),
|
repository_base=repository_base.rstrip("/"),
|
||||||
)
|
)
|
||||||
|
|
||||||
candidate.pop("signature", None)
|
|
||||||
candidate.pop("signatures", None)
|
|
||||||
candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys]
|
|
||||||
|
|
||||||
output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at)
|
output_root = resolve_output_dir(output_dir=output_dir, channel=channel, generated_at=generated_at)
|
||||||
catalog_path = output_root / "channels" / f"{channel}.json"
|
catalog_path = output_root / "channels" / f"{channel}.json"
|
||||||
keyring_path = output_root / "keyring.json"
|
keyring_path = output_root / "keyring.json"
|
||||||
summary_path = output_root / "summary.json" if write_summary else None
|
summary_path = output_root / "summary.json" if write_summary else None
|
||||||
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
keyring_payload = keyring_payload_for_candidate(
|
keyring_payload = keyring_payload_for_candidate(
|
||||||
existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json",
|
existing_keyring=web_root / "public" / "catalogs" / "v1" / "keyring.json",
|
||||||
signing_keys=parsed_keys,
|
signing_keys=parsed_keys,
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
)
|
)
|
||||||
|
release["keyring_sha256"] = canonical_hash(keyring_payload)
|
||||||
|
|
||||||
|
candidate.pop("signature", None)
|
||||||
|
candidate.pop("signatures", None)
|
||||||
|
candidate["signatures"] = [signature(candidate, key_id=key_id, private_key=private_key) for key_id, private_key in parsed_keys]
|
||||||
|
|
||||||
|
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
catalog_path.write_text(json.dumps(candidate, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
keyring_path.write_text(json.dumps(keyring_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
module_directory_files = write_module_directory(
|
module_directory_files = write_module_directory(
|
||||||
catalog_payload=candidate,
|
catalog_payload=candidate,
|
||||||
@@ -159,6 +163,18 @@ def build_selective_catalog_candidate(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def enforce_selected_version_alignment(*, repo_versions: dict[str, str], workspace: Path) -> None:
|
||||||
|
failures = [
|
||||||
|
f"{issue.repo}: {issue.source}={issue.actual!r}, expected {issue.expected!r} ({issue.message})"
|
||||||
|
for issue in selected_repository_version_issues(
|
||||||
|
repo_versions=repo_versions,
|
||||||
|
workspace=workspace,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if failures:
|
||||||
|
raise ValueError("Version alignment gate failed: " + "; ".join(failures))
|
||||||
|
|
||||||
|
|
||||||
def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str:
|
def resolve_base_catalog(*, base_catalog: Path | str | None, web_root: Path, channel: str) -> Path | str:
|
||||||
if base_catalog is not None:
|
if base_catalog is not None:
|
||||||
value = str(base_catalog)
|
value = str(base_catalog)
|
||||||
|
|||||||
@@ -76,6 +76,19 @@ def build_unit(repo: RepositorySnapshot, *, target_version: str | None, contract
|
|||||||
blockers.append(f"repository is behind {repo.upstream}")
|
blockers.append(f"repository is behind {repo.upstream}")
|
||||||
if repo.exists and repo.is_git and not repo.has_head:
|
if repo.exists and repo.is_git and not repo.has_head:
|
||||||
blockers.append("repository has no initial commit")
|
blockers.append("repository has no initial commit")
|
||||||
|
local_versions = tuple(
|
||||||
|
value
|
||||||
|
for value in (
|
||||||
|
repo.versions.pyproject,
|
||||||
|
repo.versions.package,
|
||||||
|
repo.versions.webui_package,
|
||||||
|
*repo.versions.manifests,
|
||||||
|
*repo.versions.package_inits,
|
||||||
|
)
|
||||||
|
if value
|
||||||
|
)
|
||||||
|
if len(set(local_versions)) > 1:
|
||||||
|
blockers.append("backend, manifest, and frontend version metadata is not aligned")
|
||||||
if repo.dirty:
|
if repo.dirty:
|
||||||
warnings.append("uncommitted changes will need review before release")
|
warnings.append("uncommitted changes will need review before release")
|
||||||
if repo.ahead:
|
if repo.ahead:
|
||||||
|
|||||||
652
tools/release/govoplan_release/version_alignment.py
Normal file
652
tools/release/govoplan_release/version_alignment.py
Normal file
@@ -0,0 +1,652 @@
|
|||||||
|
"""Version-alignment checks for independently released GovOPlaN packages."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
from .git_state import collect_versions
|
||||||
|
from .workspace import load_repository_specs, resolve_repo_path
|
||||||
|
|
||||||
|
|
||||||
|
_PYTHON_RELEASE_REF = re.compile(
|
||||||
|
r"^(?P<package>govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://.+/"
|
||||||
|
r"(?P<repo>govoplan-[a-z0-9-]+)\.git@v(?P<version>[^\s;]+)$"
|
||||||
|
)
|
||||||
|
_WEBUI_RELEASE_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git#v(?P<version>[^#\s]+)$")
|
||||||
|
_CATALOG_PYTHON_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git@v(?P<version>[^\s;]+)$")
|
||||||
|
_CATALOG_WEBUI_REF = re.compile(r"/(?P<repo>govoplan-[a-z0-9-]+)\.git#v(?P<version>[^#\s]+)$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class VersionAlignmentIssue:
|
||||||
|
repo: str
|
||||||
|
source: str
|
||||||
|
expected: str
|
||||||
|
actual: str
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
def repository_version_issues(
|
||||||
|
repo_path: Path,
|
||||||
|
*,
|
||||||
|
expected_version: str | None = None,
|
||||||
|
include_lockfiles: bool = True,
|
||||||
|
) -> tuple[VersionAlignmentIssue, ...]:
|
||||||
|
"""Compare every version-bearing file in one source repository."""
|
||||||
|
|
||||||
|
versions = collect_versions(repo_path)
|
||||||
|
declared = {
|
||||||
|
"pyproject.toml": versions.pyproject,
|
||||||
|
"package.json": versions.package,
|
||||||
|
"webui/package.json": versions.webui_package,
|
||||||
|
"webui/package.release.json": _json_version(repo_path / "webui" / "package.release.json")
|
||||||
|
if (repo_path / "webui" / "package.release.json").exists()
|
||||||
|
else None,
|
||||||
|
}
|
||||||
|
for index, version in enumerate(versions.manifests, start=1):
|
||||||
|
declared[f"module manifest {index}"] = version
|
||||||
|
for index, version in enumerate(versions.package_inits, start=1):
|
||||||
|
declared[f"package __init__.py {index}"] = version
|
||||||
|
|
||||||
|
canonical_source, canonical_version = next(
|
||||||
|
((source, version) for source, version in declared.items() if version),
|
||||||
|
(None, None),
|
||||||
|
)
|
||||||
|
repo = repo_path.name
|
||||||
|
if canonical_source is None or canonical_version is None:
|
||||||
|
if expected_version is None:
|
||||||
|
return ()
|
||||||
|
return (
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source="repository",
|
||||||
|
expected=expected_version.removeprefix("v"),
|
||||||
|
actual="",
|
||||||
|
message="repository has no version metadata",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = [
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=source,
|
||||||
|
expected=canonical_version,
|
||||||
|
actual=version,
|
||||||
|
message=f"{source} must match {canonical_source}",
|
||||||
|
)
|
||||||
|
for source, version in declared.items()
|
||||||
|
if version is not None and version != canonical_version
|
||||||
|
]
|
||||||
|
|
||||||
|
if expected_version is not None and canonical_version.removeprefix("v") != expected_version.removeprefix("v"):
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=canonical_source,
|
||||||
|
expected=expected_version.removeprefix("v"),
|
||||||
|
actual=canonical_version,
|
||||||
|
message="source version must match the requested release version",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if include_lockfiles:
|
||||||
|
issues.extend(
|
||||||
|
_lockfile_issues(
|
||||||
|
repo=repo,
|
||||||
|
package_path=repo_path / "package.json",
|
||||||
|
lock_path=repo_path / "package-lock.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
issues.extend(
|
||||||
|
_lockfile_issues(
|
||||||
|
repo=repo,
|
||||||
|
package_path=repo_path / "webui" / "package.json",
|
||||||
|
lock_path=repo_path / "webui" / "package-lock.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
issues.extend(
|
||||||
|
_lockfile_issues(
|
||||||
|
repo=repo,
|
||||||
|
package_path=repo_path / "webui" / "package.release.json",
|
||||||
|
lock_path=repo_path / "webui" / "package-lock.release.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def selected_repository_version_issues(
|
||||||
|
*,
|
||||||
|
repo_versions: dict[str, str],
|
||||||
|
workspace: Path,
|
||||||
|
include_lockfiles: bool = True,
|
||||||
|
) -> tuple[VersionAlignmentIssue, ...]:
|
||||||
|
"""Validate registered selected worktrees against explicit release versions."""
|
||||||
|
|
||||||
|
specs = {spec.name: spec for spec in load_repository_specs(include_website=False)}
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
for repo, expected_version in sorted(repo_versions.items()):
|
||||||
|
spec = specs.get(repo)
|
||||||
|
if spec is None:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source="repository",
|
||||||
|
expected=expected_version.removeprefix("v"),
|
||||||
|
actual="",
|
||||||
|
message="repository is not registered",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
repo_path = resolve_repo_path(spec, workspace)
|
||||||
|
if not repo_path.exists():
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source="repository",
|
||||||
|
expected=expected_version.removeprefix("v"),
|
||||||
|
actual="",
|
||||||
|
message="repository path is missing",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
issues.extend(
|
||||||
|
repository_version_issues(
|
||||||
|
repo_path,
|
||||||
|
expected_version=expected_version,
|
||||||
|
include_lockfiles=include_lockfiles,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def release_composition_issues(meta_root: Path, *, core_root: Path) -> tuple[VersionAlignmentIssue, ...]:
|
||||||
|
"""Require release references to agree and point at installable artifacts."""
|
||||||
|
|
||||||
|
python_refs = _python_release_refs(meta_root / "requirements-release.txt")
|
||||||
|
webui_refs = _webui_release_refs(core_root / "webui" / "package.release.json")
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
|
||||||
|
for repo in sorted(set(python_refs) & set(webui_refs)):
|
||||||
|
python_version = python_refs[repo]
|
||||||
|
webui_version = webui_refs[repo]
|
||||||
|
if python_version == webui_version:
|
||||||
|
continue
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source="webui/package.release.json",
|
||||||
|
expected=python_version,
|
||||||
|
actual=webui_version,
|
||||||
|
message="frontend release ref must match the backend release ref",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
workspace = meta_root.parent
|
||||||
|
for repo, version in sorted(python_refs.items()):
|
||||||
|
repo_path = workspace / repo
|
||||||
|
if repo_path.exists():
|
||||||
|
issues.extend(
|
||||||
|
_release_artifact_issues(
|
||||||
|
repo=repo,
|
||||||
|
repo_path=repo_path,
|
||||||
|
version=version,
|
||||||
|
artifact="pyproject.toml",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for repo, version in sorted(webui_refs.items()):
|
||||||
|
repo_path = workspace / repo
|
||||||
|
if repo_path.exists():
|
||||||
|
issues.extend(
|
||||||
|
_release_artifact_issues(
|
||||||
|
repo=repo,
|
||||||
|
repo_path=repo_path,
|
||||||
|
version=version,
|
||||||
|
artifact="webui/package.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def _release_artifact_issues(
|
||||||
|
*,
|
||||||
|
repo: str,
|
||||||
|
repo_path: Path,
|
||||||
|
version: str,
|
||||||
|
artifact: str,
|
||||||
|
) -> list[VersionAlignmentIssue]:
|
||||||
|
tag = f"v{version.removeprefix('v')}"
|
||||||
|
payload = _git_tag_file(repo_path, tag, artifact)
|
||||||
|
if payload is None:
|
||||||
|
return [
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{tag}:{artifact}",
|
||||||
|
expected="installable package metadata",
|
||||||
|
actual="missing or unreadable",
|
||||||
|
message="release tag must contain its declared package artifact",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
if artifact == "pyproject.toml":
|
||||||
|
parsed = tomllib.loads(payload)
|
||||||
|
project = parsed.get("project")
|
||||||
|
artifact_version = project.get("version") if isinstance(project, dict) else None
|
||||||
|
else:
|
||||||
|
parsed = json.loads(payload)
|
||||||
|
artifact_version = parsed.get("version") if isinstance(parsed, dict) else None
|
||||||
|
except (json.JSONDecodeError, tomllib.TOMLDecodeError):
|
||||||
|
artifact_version = None
|
||||||
|
if artifact_version == version.removeprefix("v"):
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{tag}:{artifact}",
|
||||||
|
expected=version.removeprefix("v"),
|
||||||
|
actual=str(artifact_version or ""),
|
||||||
|
message="release artifact version must match its immutable tag",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def candidate_catalog_version_issues(payload: object) -> tuple[VersionAlignmentIssue, ...]:
|
||||||
|
"""Validate versions, refs, and selected-unit provenance inside a catalog."""
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return (
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo="catalog",
|
||||||
|
source="catalog",
|
||||||
|
expected="object",
|
||||||
|
actual=type(payload).__name__,
|
||||||
|
message="candidate catalog must be a JSON object",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
represented: dict[str, str] = {}
|
||||||
|
core_release = payload.get("core_release")
|
||||||
|
if isinstance(core_release, dict):
|
||||||
|
issues.extend(_catalog_entry_issues(core_release, source="core_release", represented=represented))
|
||||||
|
else:
|
||||||
|
issues.append(_catalog_shape_issue("core_release", "candidate catalog has no core release"))
|
||||||
|
|
||||||
|
modules = payload.get("modules")
|
||||||
|
if not isinstance(modules, list):
|
||||||
|
issues.append(_catalog_shape_issue("modules", "candidate catalog has no modules list"))
|
||||||
|
else:
|
||||||
|
for index, entry in enumerate(modules):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
issues.append(_catalog_shape_issue(f"modules[{index}]", "module entry must be an object"))
|
||||||
|
continue
|
||||||
|
issues.extend(
|
||||||
|
_catalog_entry_issues(
|
||||||
|
entry,
|
||||||
|
source=f"modules[{index}]",
|
||||||
|
represented=represented,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
release = payload.get("release")
|
||||||
|
if isinstance(release, dict):
|
||||||
|
issues.extend(_catalog_release_issues(release, represented=represented))
|
||||||
|
return tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_entry_issues(
|
||||||
|
entry: dict[str, object],
|
||||||
|
*,
|
||||||
|
source: str,
|
||||||
|
represented: dict[str, str],
|
||||||
|
) -> list[VersionAlignmentIssue]:
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
version = entry.get("version")
|
||||||
|
normalized_version = version.removeprefix("v") if isinstance(version, str) and version else None
|
||||||
|
if normalized_version is None:
|
||||||
|
issues.append(_catalog_shape_issue(f"{source}.version", "catalog entry has no version"))
|
||||||
|
|
||||||
|
python_ref = entry.get("python_ref")
|
||||||
|
python_match = _CATALOG_PYTHON_REF.search(python_ref) if isinstance(python_ref, str) else None
|
||||||
|
if python_match is None:
|
||||||
|
issues.append(_catalog_shape_issue(f"{source}.python_ref", "Python ref must end in a version tag"))
|
||||||
|
repo = str(entry.get("python_package") or entry.get("module_id") or source)
|
||||||
|
else:
|
||||||
|
repo = python_match.group("repo")
|
||||||
|
ref_version = python_match.group("version").removeprefix("v")
|
||||||
|
if normalized_version is not None and ref_version != normalized_version:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{source}.python_ref",
|
||||||
|
expected=normalized_version,
|
||||||
|
actual=ref_version,
|
||||||
|
message="Python ref tag must match the catalog entry version",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if normalized_version is not None:
|
||||||
|
previous = represented.setdefault(repo, normalized_version)
|
||||||
|
if previous != normalized_version:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=source,
|
||||||
|
expected=previous,
|
||||||
|
actual=normalized_version,
|
||||||
|
message="repository appears with conflicting catalog versions",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
webui_package = entry.get("webui_package")
|
||||||
|
webui_ref = entry.get("webui_ref")
|
||||||
|
if bool(webui_package) != bool(webui_ref):
|
||||||
|
issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI package and ref must be declared together"))
|
||||||
|
if isinstance(webui_ref, str):
|
||||||
|
webui_match = _CATALOG_WEBUI_REF.search(webui_ref)
|
||||||
|
if webui_match is None:
|
||||||
|
issues.append(_catalog_shape_issue(f"{source}.webui_ref", "WebUI ref must end in a version tag"))
|
||||||
|
else:
|
||||||
|
webui_repo = webui_match.group("repo")
|
||||||
|
webui_version = webui_match.group("version").removeprefix("v")
|
||||||
|
if python_match is not None and webui_repo != repo:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{source}.webui_ref",
|
||||||
|
expected=repo,
|
||||||
|
actual=webui_repo,
|
||||||
|
message="backend and WebUI refs must use the same repository",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if normalized_version is not None and webui_version != normalized_version:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{source}.webui_ref",
|
||||||
|
expected=normalized_version,
|
||||||
|
actual=webui_version,
|
||||||
|
message="WebUI ref tag must match the catalog entry version",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_release_issues(
|
||||||
|
release: dict[str, object],
|
||||||
|
*,
|
||||||
|
represented: dict[str, str],
|
||||||
|
) -> list[VersionAlignmentIssue]:
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
version = release.get("version")
|
||||||
|
tag = release.get("tag")
|
||||||
|
if isinstance(version, str) and isinstance(tag, str) and tag != f"v{version.removeprefix('v')}":
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo="govoplan-core",
|
||||||
|
source="release.tag",
|
||||||
|
expected=f"v{version.removeprefix('v')}",
|
||||||
|
actual=tag,
|
||||||
|
message="release tag must match release version",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
selected_units = release.get("selected_units")
|
||||||
|
if selected_units is None:
|
||||||
|
return issues
|
||||||
|
if not isinstance(selected_units, list):
|
||||||
|
issues.append(_catalog_shape_issue("release.selected_units", "selected units must be a list"))
|
||||||
|
return issues
|
||||||
|
seen: set[str] = set()
|
||||||
|
for index, unit in enumerate(selected_units):
|
||||||
|
if not isinstance(unit, dict):
|
||||||
|
issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit must be an object"))
|
||||||
|
continue
|
||||||
|
repo = unit.get("repo")
|
||||||
|
unit_version = unit.get("version")
|
||||||
|
unit_tag = unit.get("tag")
|
||||||
|
if not isinstance(repo, str) or not isinstance(unit_version, str) or not isinstance(unit_tag, str):
|
||||||
|
issues.append(_catalog_shape_issue(f"release.selected_units[{index}]", "selected unit requires repo, version, and tag"))
|
||||||
|
continue
|
||||||
|
normalized_version = unit_version.removeprefix("v")
|
||||||
|
if repo in seen:
|
||||||
|
issues.append(_catalog_shape_issue(f"release.selected_units[{index}].repo", "selected repository is duplicated"))
|
||||||
|
seen.add(repo)
|
||||||
|
if unit_tag != f"v{normalized_version}":
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"release.selected_units[{index}].tag",
|
||||||
|
expected=f"v{normalized_version}",
|
||||||
|
actual=unit_tag,
|
||||||
|
message="selected-unit tag must match its version",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
represented_version = represented.get(repo)
|
||||||
|
if represented_version is None:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"release.selected_units[{index}]",
|
||||||
|
expected=normalized_version,
|
||||||
|
actual="",
|
||||||
|
message="selected repository is not represented in the catalog",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif represented_version != normalized_version:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"release.selected_units[{index}].version",
|
||||||
|
expected=represented_version,
|
||||||
|
actual=normalized_version,
|
||||||
|
message="selected-unit version must match the catalog entry",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_shape_issue(source: str, message: str) -> VersionAlignmentIssue:
|
||||||
|
return VersionAlignmentIssue(
|
||||||
|
repo="catalog",
|
||||||
|
source=source,
|
||||||
|
expected="valid value",
|
||||||
|
actual="",
|
||||||
|
message=message,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _lockfile_issues(*, repo: str, package_path: Path, lock_path: Path) -> list[VersionAlignmentIssue]:
|
||||||
|
if not package_path.exists() or not lock_path.exists():
|
||||||
|
return []
|
||||||
|
package_payload = _json_object(package_path)
|
||||||
|
lock_payload = _json_object(lock_path)
|
||||||
|
package_version = _payload_version(package_payload)
|
||||||
|
lock_version = _lockfile_root_version(lock_path)
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
if package_version is None or lock_version is None or package_version == lock_version:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=str(lock_path.relative_to(package_path.parents[1] if package_path.parent.name == "webui" else package_path.parent)),
|
||||||
|
expected=package_version,
|
||||||
|
actual=lock_version,
|
||||||
|
message=f"lockfile root version must match {package_path.name}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
lock_packages = lock_payload.get("packages")
|
||||||
|
lock_root = lock_packages.get("") if isinstance(lock_packages, dict) else None
|
||||||
|
if isinstance(lock_root, dict):
|
||||||
|
for group in ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies"):
|
||||||
|
package_dependencies = package_payload.get(group) or {}
|
||||||
|
lock_dependencies = lock_root.get(group) or {}
|
||||||
|
if package_dependencies != lock_dependencies:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{lock_path.name}:{group}",
|
||||||
|
expected=json.dumps(package_dependencies, sort_keys=True),
|
||||||
|
actual=json.dumps(lock_dependencies, sort_keys=True),
|
||||||
|
message=f"lockfile root {group} must match {package_path.name}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
issues.extend(
|
||||||
|
_git_lock_resolution_issues(
|
||||||
|
repo=repo,
|
||||||
|
package_payload=package_payload,
|
||||||
|
lock_packages=lock_packages,
|
||||||
|
workspace=package_path.parents[2] if package_path.parent.name == "webui" else package_path.parent.parent,
|
||||||
|
lock_name=lock_path.name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def _git_lock_resolution_issues(
|
||||||
|
*,
|
||||||
|
repo: str,
|
||||||
|
package_payload: dict[str, object],
|
||||||
|
lock_packages: dict[str, object],
|
||||||
|
workspace: Path,
|
||||||
|
lock_name: str,
|
||||||
|
) -> list[VersionAlignmentIssue]:
|
||||||
|
dependencies = package_payload.get("dependencies")
|
||||||
|
if not isinstance(dependencies, dict):
|
||||||
|
return []
|
||||||
|
issues: list[VersionAlignmentIssue] = []
|
||||||
|
for package_name, spec in dependencies.items():
|
||||||
|
if not isinstance(package_name, str) or not isinstance(spec, str):
|
||||||
|
continue
|
||||||
|
match = _CATALOG_WEBUI_REF.search(spec)
|
||||||
|
if match is None:
|
||||||
|
continue
|
||||||
|
source_repo = match.group("repo")
|
||||||
|
version = match.group("version").removeprefix("v")
|
||||||
|
locked = lock_packages.get(f"node_modules/{package_name}")
|
||||||
|
if not isinstance(locked, dict):
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{lock_name}:{package_name}",
|
||||||
|
expected=version,
|
||||||
|
actual="",
|
||||||
|
message="release lock has no resolved GovOPlaN package",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
locked_version = locked.get("version")
|
||||||
|
if locked_version != version:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{lock_name}:{package_name}",
|
||||||
|
expected=version,
|
||||||
|
actual=str(locked_version or ""),
|
||||||
|
message="resolved package version must match its release ref",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
expected_commit = _git_tag_commit(workspace / source_repo, f"v{version}")
|
||||||
|
resolved = locked.get("resolved")
|
||||||
|
resolved_commit = resolved.rsplit("#", 1)[-1] if isinstance(resolved, str) and "#" in resolved else None
|
||||||
|
if expected_commit is None or resolved_commit != expected_commit:
|
||||||
|
issues.append(
|
||||||
|
VersionAlignmentIssue(
|
||||||
|
repo=repo,
|
||||||
|
source=f"{lock_name}:{package_name}:resolved",
|
||||||
|
expected=expected_commit or f"local tag v{version}",
|
||||||
|
actual=resolved_commit or "",
|
||||||
|
message="resolved package commit must match the local release tag",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def _git_tag_commit(repo_path: Path, tag: str) -> str | None:
|
||||||
|
if not (repo_path / ".git").exists():
|
||||||
|
return None
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(repo_path), "rev-list", "-n", "1", tag],
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
value = result.stdout.strip()
|
||||||
|
return value if result.returncode == 0 and value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _git_tag_file(repo_path: Path, tag: str, relative_path: str) -> str | None:
|
||||||
|
if not (repo_path / ".git").exists():
|
||||||
|
return None
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(repo_path), "show", f"{tag}:{relative_path}"],
|
||||||
|
check=False,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
return result.stdout if result.returncode == 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _json_version(path: Path) -> str | None:
|
||||||
|
value = _payload_version(_json_object(path))
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_version(payload: dict[str, object]) -> str | None:
|
||||||
|
value = payload.get("version")
|
||||||
|
return value if isinstance(value, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _json_object(path: Path) -> dict[str, object]:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
return payload if isinstance(payload, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _lockfile_root_version(path: Path) -> str | None:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
|
packages = payload.get("packages")
|
||||||
|
if isinstance(packages, dict):
|
||||||
|
root = packages.get("")
|
||||||
|
if isinstance(root, dict) and isinstance(root.get("version"), str):
|
||||||
|
return root["version"]
|
||||||
|
value = payload.get("version")
|
||||||
|
return value if isinstance(value, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _python_release_refs(path: Path) -> dict[str, str]:
|
||||||
|
if not path.exists():
|
||||||
|
return {}
|
||||||
|
refs: dict[str, str] = {}
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
match = _PYTHON_RELEASE_REF.match(line.strip())
|
||||||
|
if match is None:
|
||||||
|
continue
|
||||||
|
repo = match.group("repo")
|
||||||
|
refs[repo] = match.group("version")
|
||||||
|
return refs
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_release_refs(path: Path) -> dict[str, str]:
|
||||||
|
if not path.exists():
|
||||||
|
return {}
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
dependencies = payload.get("dependencies") if isinstance(payload, dict) else None
|
||||||
|
if not isinstance(dependencies, dict):
|
||||||
|
return {}
|
||||||
|
refs: dict[str, str] = {}
|
||||||
|
for spec in dependencies.values():
|
||||||
|
if not isinstance(spec, str):
|
||||||
|
continue
|
||||||
|
match = _WEBUI_RELEASE_REF.search(spec)
|
||||||
|
if match is not None:
|
||||||
|
refs[match.group("repo")] = match.group("version")
|
||||||
|
return refs
|
||||||
@@ -194,6 +194,7 @@ run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
GEN_ARGS=(
|
GEN_ARGS=(
|
||||||
|
env "GOVOPLAN_CORE_ROOT=$CORE_ROOT"
|
||||||
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
|
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
|
||||||
--version "$VERSION"
|
--version "$VERSION"
|
||||||
--channel "$CHANNEL"
|
--channel "$CHANNEL"
|
||||||
|
|||||||
@@ -52,28 +52,10 @@ Options:
|
|||||||
-h, --help Show this help.
|
-h, --help Show this help.
|
||||||
|
|
||||||
Repos:
|
Repos:
|
||||||
Installable release packages:
|
Core, every registered release/module repository listed below in this script,
|
||||||
govoplan-access
|
and the meta repository. Repositories with pyproject.toml are versioned and
|
||||||
govoplan-admin
|
alignment-gated dynamically; remaining roadmap/scaffold repositories are
|
||||||
govoplan-tenancy
|
committed and tagged without inventing package metadata.
|
||||||
govoplan-organizations
|
|
||||||
govoplan-identity
|
|
||||||
govoplan-idm
|
|
||||||
govoplan-policy
|
|
||||||
govoplan-audit
|
|
||||||
govoplan-dashboard
|
|
||||||
govoplan-files
|
|
||||||
govoplan-mail
|
|
||||||
govoplan-campaign
|
|
||||||
govoplan-calendar
|
|
||||||
govoplan-ops
|
|
||||||
govoplan-core
|
|
||||||
|
|
||||||
Roadmap/scaffold module repositories are tag-only until they have package
|
|
||||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
|
||||||
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
|
||||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
|
||||||
xrechnung, and xta-osci.
|
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
tools/release/push-release-tag.sh
|
tools/release/push-release-tag.sh
|
||||||
@@ -121,12 +103,14 @@ PACKAGE_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-organizations"
|
"$PARENT/govoplan-organizations"
|
||||||
"$PARENT/govoplan-permits"
|
"$PARENT/govoplan-permits"
|
||||||
"$PARENT/govoplan-policy"
|
"$PARENT/govoplan-policy"
|
||||||
|
"$PARENT/govoplan-poll"
|
||||||
"$PARENT/govoplan-procurement"
|
"$PARENT/govoplan-procurement"
|
||||||
"$PARENT/govoplan-records"
|
"$PARENT/govoplan-records"
|
||||||
"$PARENT/govoplan-resources"
|
"$PARENT/govoplan-resources"
|
||||||
"$PARENT/govoplan-risk-compliance"
|
"$PARENT/govoplan-risk-compliance"
|
||||||
"$PARENT/govoplan-tenancy"
|
"$PARENT/govoplan-tenancy"
|
||||||
"$PARENT/govoplan-transparency"
|
"$PARENT/govoplan-transparency"
|
||||||
|
"$PARENT/govoplan-evaluation"
|
||||||
)
|
)
|
||||||
TAG_ONLY_MODULE_REPOS=(
|
TAG_ONLY_MODULE_REPOS=(
|
||||||
"$PARENT/govoplan-addresses"
|
"$PARENT/govoplan-addresses"
|
||||||
@@ -134,6 +118,7 @@ TAG_ONLY_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-cases"
|
"$PARENT/govoplan-cases"
|
||||||
"$PARENT/govoplan-connectors"
|
"$PARENT/govoplan-connectors"
|
||||||
"$PARENT/govoplan-dms"
|
"$PARENT/govoplan-dms"
|
||||||
|
"$PARENT/govoplan-dist-lists"
|
||||||
"$PARENT/govoplan-erp"
|
"$PARENT/govoplan-erp"
|
||||||
"$PARENT/govoplan-fit-connect"
|
"$PARENT/govoplan-fit-connect"
|
||||||
"$PARENT/govoplan-forms"
|
"$PARENT/govoplan-forms"
|
||||||
@@ -144,8 +129,10 @@ TAG_ONLY_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-portal"
|
"$PARENT/govoplan-portal"
|
||||||
"$PARENT/govoplan-postbox"
|
"$PARENT/govoplan-postbox"
|
||||||
"$PARENT/govoplan-reporting"
|
"$PARENT/govoplan-reporting"
|
||||||
|
"$PARENT/govoplan-rest"
|
||||||
"$PARENT/govoplan-scheduling"
|
"$PARENT/govoplan-scheduling"
|
||||||
"$PARENT/govoplan-search"
|
"$PARENT/govoplan-search"
|
||||||
|
"$PARENT/govoplan-soap"
|
||||||
"$PARENT/govoplan-tasks"
|
"$PARENT/govoplan-tasks"
|
||||||
"$PARENT/govoplan-templates"
|
"$PARENT/govoplan-templates"
|
||||||
"$PARENT/govoplan-workflow"
|
"$PARENT/govoplan-workflow"
|
||||||
@@ -154,10 +141,12 @@ TAG_ONLY_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-xta-osci"
|
"$PARENT/govoplan-xta-osci"
|
||||||
)
|
)
|
||||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
SUPPORT_REPOS=("$META_ROOT")
|
||||||
|
PACKAGE_REPOS=()
|
||||||
REPOS=(
|
REPOS=(
|
||||||
"${MODULE_REPOS[@]}"
|
"${MODULE_REPOS[@]}"
|
||||||
"$ROOT"
|
"$ROOT"
|
||||||
|
"${SUPPORT_REPOS[@]}"
|
||||||
)
|
)
|
||||||
|
|
||||||
REMOTE="origin"
|
REMOTE="origin"
|
||||||
@@ -379,6 +368,7 @@ if project_name != "govoplan-core":
|
|||||||
peers["@govoplan/core-webui"] = f"^{new_version}"
|
peers["@govoplan/core-webui"] = f"^{new_version}"
|
||||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||||
PYCODE
|
PYCODE
|
||||||
|
synchronize_lockfile_root "$package_path" "${package_path%package.json}package-lock.json"
|
||||||
done
|
done
|
||||||
|
|
||||||
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
||||||
@@ -401,9 +391,59 @@ for name, spec in list(dependencies.items()):
|
|||||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||||
PYCODE
|
PYCODE
|
||||||
|
synchronize_lockfile_root "$release_package_path" "$repo/webui/package-lock.release.json"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
synchronize_lockfile_root() {
|
||||||
|
local package_path="$1"
|
||||||
|
local lock_path="$2"
|
||||||
|
[[ -f "$package_path" && -f "$lock_path" ]] || return 0
|
||||||
|
"$PYTHON" - "$package_path" "$lock_path" <<'PYCODE'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
package_path = Path(sys.argv[1])
|
||||||
|
lock_path = Path(sys.argv[2])
|
||||||
|
package = json.loads(package_path.read_text())
|
||||||
|
lock = json.loads(lock_path.read_text())
|
||||||
|
version = package.get("version")
|
||||||
|
if not isinstance(version, str) or not version:
|
||||||
|
raise SystemExit(f"package version is missing from {package_path}")
|
||||||
|
lock["version"] = version
|
||||||
|
packages = lock.get("packages")
|
||||||
|
if isinstance(packages, dict) and isinstance(packages.get(""), dict):
|
||||||
|
packages[""]["version"] = version
|
||||||
|
lock_path.write_text(json.dumps(lock, indent=2) + "\n")
|
||||||
|
PYCODE
|
||||||
|
}
|
||||||
|
|
||||||
|
update_release_requirements() {
|
||||||
|
local version="$1"
|
||||||
|
"$PYTHON" - "$META_ROOT/requirements-release.txt" "$version" <<'PYCODE'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path = Path(sys.argv[1])
|
||||||
|
version = sys.argv[2]
|
||||||
|
text = path.read_text()
|
||||||
|
updated, count = re.subn(
|
||||||
|
r"(?m)^(govoplan-[a-z0-9-]+\s+@\s+git\+ssh://[^\s]+\.git)@v[^\s]+$",
|
||||||
|
rf"\1@v{version}",
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
if count == 0:
|
||||||
|
raise SystemExit(f"no tagged GovOPlaN requirements found in {path}")
|
||||||
|
path.write_text(updated)
|
||||||
|
PYCODE
|
||||||
|
}
|
||||||
|
|
||||||
update_version_files() {
|
update_version_files() {
|
||||||
local repo="$1"
|
local repo="$1"
|
||||||
local version="$2"
|
local version="$2"
|
||||||
@@ -427,7 +467,30 @@ generate_release_lock() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
[[ -x "$META_ROOT/tools/release/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $META_ROOT/tools/release/generate-release-lock.sh"
|
[[ -x "$META_ROOT/tools/release/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $META_ROOT/tools/release/generate-release-lock.sh"
|
||||||
run "$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN"
|
local command=("$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN")
|
||||||
|
local repo
|
||||||
|
for repo in "${MODULE_REPOS[@]}"; do
|
||||||
|
command+=(--local-git-repo "$repo")
|
||||||
|
done
|
||||||
|
run "${command[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_version_alignment_gate() {
|
||||||
|
local mode="${1:-full}"
|
||||||
|
local command=(
|
||||||
|
"$PYTHON"
|
||||||
|
"$META_ROOT/tools/checks/check-version-alignment.py"
|
||||||
|
--workspace-root "$PARENT"
|
||||||
|
--release-composition
|
||||||
|
)
|
||||||
|
if [[ "$mode" == "source" ]]; then
|
||||||
|
command+=(--source-metadata-only)
|
||||||
|
fi
|
||||||
|
local repo
|
||||||
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
|
command+=(--repo-version "$(basename "$repo")=$TARGET_VERSION")
|
||||||
|
done
|
||||||
|
run "${command[@]}"
|
||||||
}
|
}
|
||||||
|
|
||||||
run_migration_release_audit() {
|
run_migration_release_audit() {
|
||||||
@@ -694,6 +757,15 @@ for repo in "${REPOS[@]}"; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
EFFECTIVE_TAG_ONLY_REPOS=()
|
||||||
|
for repo in "${REPOS[@]}"; do
|
||||||
|
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||||
|
PACKAGE_REPOS+=("$repo")
|
||||||
|
else
|
||||||
|
EFFECTIVE_TAG_ONLY_REPOS+=("$repo")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
if [[ -z "$TARGET_VERSION" ]]; then
|
if [[ -z "$TARGET_VERSION" ]]; then
|
||||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
@@ -744,9 +816,9 @@ echo " tag: $TAG"
|
|||||||
echo " remote: $REMOTE"
|
echo " remote: $REMOTE"
|
||||||
echo " commit message: $COMMIT_MESSAGE"
|
echo " commit message: $COMMIT_MESSAGE"
|
||||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
echo " tag-only/support repos: ${#EFFECTIVE_TAG_ONLY_REPOS[@]} committed/tagged/pushed without package version files"
|
||||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
echo " release lock: resolve core webui/package-lock.release.json from local reviewed module tags"
|
||||||
else
|
else
|
||||||
echo " release lock: skipped"
|
echo " release lock: skipped"
|
||||||
fi
|
fi
|
||||||
@@ -780,9 +852,21 @@ for repo in "${PACKAGE_REPOS[@]}"; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
|
echo "Would update $META_ROOT/requirements-release.txt to $TAG"
|
||||||
|
else
|
||||||
|
update_release_requirements "$TARGET_VERSION"
|
||||||
|
fi
|
||||||
|
|
||||||
refresh_development_webui_lock
|
refresh_development_webui_lock
|
||||||
|
|
||||||
for repo in "${MODULE_REPOS[@]}"; do
|
# Source metadata, release refs, and lock roots must agree before creating any
|
||||||
|
# release commit or tag. A second gate below verifies the fully resolved release
|
||||||
|
# lock before any remote mutation.
|
||||||
|
run_version_alignment_gate source
|
||||||
|
|
||||||
|
PRE_CORE_REPOS=("${MODULE_REPOS[@]}" "${SUPPORT_REPOS[@]}")
|
||||||
|
for repo in "${PRE_CORE_REPOS[@]}"; do
|
||||||
run git -C "$repo" add -A
|
run git -C "$repo" add -A
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
@@ -806,12 +890,13 @@ for repo in "${MODULE_REPOS[@]}"; do
|
|||||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||||
done
|
done
|
||||||
|
|
||||||
for repo in "${MODULE_REPOS[@]}"; do
|
|
||||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
|
||||||
done
|
|
||||||
|
|
||||||
generate_release_lock
|
generate_release_lock
|
||||||
|
|
||||||
|
# npm resolves the new module tags from the local repositories. Nothing has
|
||||||
|
# been pushed yet; a failed lock or alignment check leaves only local,
|
||||||
|
# recoverable commits/tags.
|
||||||
|
run_version_alignment_gate
|
||||||
|
|
||||||
run git -C "$ROOT" add -A
|
run git -C "$ROOT" add -A
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
@@ -822,6 +907,10 @@ fi
|
|||||||
|
|
||||||
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||||
|
|
||||||
|
for repo in "${PRE_CORE_REPOS[@]}"; do
|
||||||
|
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||||
|
done
|
||||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||||
|
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||||
|
|||||||
@@ -381,7 +381,7 @@ def repo_findings(repos: tuple[RepoState, ...], *, target_version: str | None, t
|
|||||||
findings.append(
|
findings.append(
|
||||||
Finding(
|
Finding(
|
||||||
check_id="repo-version-mismatch",
|
check_id="repo-version-mismatch",
|
||||||
severity="warning",
|
severity="blocker",
|
||||||
title=f"{repo.name} has inconsistent local version files",
|
title=f"{repo.name} has inconsistent local version files",
|
||||||
detail=", ".join(
|
detail=", ".join(
|
||||||
value
|
value
|
||||||
|
|||||||
Reference in New Issue
Block a user