Compare commits
56
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f039dd39c | ||
|
|
d107d94fec | ||
|
|
6163c5992f | ||
|
|
2f28f22fd1 | ||
|
|
909862afdb | ||
|
|
eb04804d36 | ||
|
|
017aa7a702 | ||
|
|
af27b9fbdf | ||
|
|
cb45251c59 | ||
|
|
3b3d5b3386 | ||
|
|
313249b8fc | ||
|
|
282c90c54b | ||
|
|
25424187a8 | ||
|
|
ff8ee991c3 | ||
|
|
a5a0731d20 | ||
|
|
a0f161041d | ||
|
|
cb85999a14 | ||
|
|
cbfe8b03a7 | ||
|
|
768e9a51c9 | ||
|
|
087561ee12 | ||
|
|
11c1aa1815 | ||
|
|
478ecb5d0e | ||
|
|
32689d027a | ||
|
|
b7cc2d2df4 | ||
|
|
b70869e747 | ||
|
|
0c84afb158 | ||
|
|
145aa58c11 | ||
|
|
a3566c9311 | ||
|
|
3219460064 | ||
|
|
4b2a15adb5 | ||
|
|
acc5ffc247 | ||
|
|
f4f9836a09 | ||
|
|
758fa1bba7 | ||
|
|
0acc8cfc31 | ||
|
|
344bcaf1bc | ||
|
|
794622e4ed | ||
|
|
7653e9851f | ||
|
|
6f896d9c04 | ||
|
|
8f5ac52b58 | ||
|
|
2bc9ad7f00 | ||
|
|
fa1a4bacfb | ||
|
|
0c0669768d | ||
|
|
7b6ceeb185 | ||
|
|
ff12f676a1 | ||
|
|
eb9ab9ef1c | ||
|
|
935c1fe162 | ||
|
|
c7d1cd0e8f | ||
|
|
ac80d7e4e3 | ||
|
|
5e449b0983 | ||
|
|
d4bf07b446 | ||
|
|
adc4db9fdf | ||
|
|
484f2af3ac | ||
|
|
abf9564cee | ||
|
|
5bef966119 | ||
|
|
5e80b39bbd | ||
|
|
9370f501a0 |
@@ -39,6 +39,10 @@ on:
|
|||||||
description: Digest-pinned GreenMail image
|
description: Digest-pinned GreenMail image
|
||||||
required: true
|
required: true
|
||||||
type: string
|
type: string
|
||||||
|
binfmt_image:
|
||||||
|
description: Digest-pinned tonistiigi/binfmt image for arm64 CI execution
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
publish-runtime:
|
publish-runtime:
|
||||||
@@ -53,6 +57,41 @@ jobs:
|
|||||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||||
with:
|
with:
|
||||||
node-version: "22"
|
node-version: "22"
|
||||||
|
- name: Validate immutable release inputs
|
||||||
|
env:
|
||||||
|
VERSION: ${{ inputs.version }}
|
||||||
|
PYTHON_IMAGE: ${{ inputs.python_image }}
|
||||||
|
NGINX_IMAGE: ${{ inputs.nginx_image }}
|
||||||
|
POSTGRES_IMAGE: ${{ inputs.postgres_image }}
|
||||||
|
REDIS_IMAGE: ${{ inputs.redis_image }}
|
||||||
|
LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }}
|
||||||
|
MANAGED_INGRESS_IMAGE: ${{ inputs.managed_ingress_image }}
|
||||||
|
GARAGE_IMAGE: ${{ inputs.garage_image }}
|
||||||
|
TEST_MAIL_IMAGE: ${{ inputs.test_mail_image }}
|
||||||
|
BINFMT_IMAGE: ${{ inputs.binfmt_image }}
|
||||||
|
run: |
|
||||||
|
python - <<'PY'
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
version = os.environ["VERSION"]
|
||||||
|
if re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?", version) is None:
|
||||||
|
raise SystemExit("version must be a SemVer value without a leading v")
|
||||||
|
image_pattern = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
|
||||||
|
for name in (
|
||||||
|
"PYTHON_IMAGE",
|
||||||
|
"NGINX_IMAGE",
|
||||||
|
"POSTGRES_IMAGE",
|
||||||
|
"REDIS_IMAGE",
|
||||||
|
"LOAD_BALANCER_IMAGE",
|
||||||
|
"MANAGED_INGRESS_IMAGE",
|
||||||
|
"GARAGE_IMAGE",
|
||||||
|
"TEST_MAIL_IMAGE",
|
||||||
|
"BINFMT_IMAGE",
|
||||||
|
):
|
||||||
|
if image_pattern.fullmatch(os.environ[name]) is None:
|
||||||
|
raise SystemExit(f"{name} must be an exact sha256 image reference")
|
||||||
|
PY
|
||||||
- name: Use HTTPS for GovOPlaN repositories
|
- name: Use HTTPS for GovOPlaN repositories
|
||||||
run: |
|
run: |
|
||||||
git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan"
|
git config --global --add url."https://git.add-ideas.de/GovOPlaN/govoplan".insteadOf "git@git.add-ideas.de:GovOPlaN/govoplan"
|
||||||
@@ -162,6 +201,41 @@ jobs:
|
|||||||
WEB_DIGEST="sha256:$(sha256sum runtime-output/web-index.json | cut -d' ' -f1)"
|
WEB_DIGEST="sha256:$(sha256sum runtime-output/web-index.json | cut -d' ' -f1)"
|
||||||
python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-api --index-digest "$API_DIGEST" --index runtime-output/api-index.json --output runtime-output/api-metadata.json
|
python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-api --index-digest "$API_DIGEST" --index runtime-output/api-index.json --output runtime-output/api-metadata.json
|
||||||
python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-web --index-digest "$WEB_DIGEST" --index runtime-output/web-index.json --output runtime-output/web-metadata.json
|
python tools/release/resolve-oci-platforms.py --repository git.add-ideas.de/govoplan/runtime-web --index-digest "$WEB_DIGEST" --index runtime-output/web-index.json --output runtime-output/web-metadata.json
|
||||||
|
- name: Resolve managed dependency platform images
|
||||||
|
working-directory: govoplan
|
||||||
|
env:
|
||||||
|
POSTGRES_IMAGE: ${{ inputs.postgres_image }}
|
||||||
|
REDIS_IMAGE: ${{ inputs.redis_image }}
|
||||||
|
run: |
|
||||||
|
docker buildx imagetools inspect "$POSTGRES_IMAGE" --raw > runtime-output/postgres-index.json
|
||||||
|
docker buildx imagetools inspect "$REDIS_IMAGE" --raw > runtime-output/redis-index.json
|
||||||
|
python tools/release/resolve-oci-platforms.py \
|
||||||
|
--repository "${POSTGRES_IMAGE%@*}" \
|
||||||
|
--index-digest "${POSTGRES_IMAGE##*@}" \
|
||||||
|
--index runtime-output/postgres-index.json \
|
||||||
|
--output runtime-output/postgres-metadata.json
|
||||||
|
python tools/release/resolve-oci-platforms.py \
|
||||||
|
--repository "${REDIS_IMAGE%@*}" \
|
||||||
|
--index-digest "${REDIS_IMAGE##*@}" \
|
||||||
|
--index runtime-output/redis-index.json \
|
||||||
|
--output runtime-output/redis-metadata.json
|
||||||
|
- name: Register arm64 execution for runtime smoke
|
||||||
|
working-directory: govoplan
|
||||||
|
env:
|
||||||
|
BINFMT_IMAGE: ${{ inputs.binfmt_image }}
|
||||||
|
run: docker run --privileged --rm "$BINFMT_IMAGE" --install arm64
|
||||||
|
- name: Exercise amd64 and arm64 runtime images
|
||||||
|
working-directory: govoplan
|
||||||
|
run: |
|
||||||
|
for ARCH in amd64 arm64; do
|
||||||
|
.runtime-build/bin/python tools/checks/runtime-image-smoke.py \
|
||||||
|
--api-metadata runtime-output/api-metadata.json \
|
||||||
|
--web-metadata runtime-output/web-metadata.json \
|
||||||
|
--postgres-metadata runtime-output/postgres-metadata.json \
|
||||||
|
--redis-metadata runtime-output/redis-metadata.json \
|
||||||
|
--platform "linux/$ARCH" \
|
||||||
|
--output "runtime-output/evidence/runtime-smoke-$ARCH.json"
|
||||||
|
done
|
||||||
- name: Generate and sign distribution evidence
|
- name: Generate and sign distribution evidence
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
env:
|
env:
|
||||||
@@ -202,21 +276,53 @@ jobs:
|
|||||||
--dependency "test_mail=$TEST_MAIL_IMAGE" \
|
--dependency "test_mail=$TEST_MAIL_IMAGE" \
|
||||||
--output-directory runtime-output/evidence \
|
--output-directory runtime-output/evidence \
|
||||||
--descriptor runtime-output/distribution-descriptor.json
|
--descriptor runtime-output/distribution-descriptor.json
|
||||||
python tools/release/generate-runtime-distribution.py \
|
.runtime-build/bin/python tools/release/generate-runtime-distribution.py \
|
||||||
--descriptor runtime-output/distribution-descriptor.json \
|
--descriptor runtime-output/distribution-descriptor.json \
|
||||||
--signing-key "$SIGNING_KEY_ID=runtime-output/signing-key.pem" \
|
--signing-key "$SIGNING_KEY_ID=runtime-output/signing-key.pem" \
|
||||||
--output runtime-output/distribution-manifest.json
|
--output runtime-output/distribution-manifest.json
|
||||||
openssl pkeyutl -sign -inkey runtime-output/signing-key.pem -rawin \
|
openssl pkeyutl -sign -inkey runtime-output/signing-key.pem -rawin \
|
||||||
-in runtime-output/govoplan-deploy.pyz \
|
-in runtime-output/govoplan-deploy.pyz \
|
||||||
-out runtime-output/govoplan-deploy.pyz.sig
|
-out runtime-output/govoplan-deploy.pyz.sig
|
||||||
sha256sum runtime-output/govoplan-deploy.pyz > runtime-output/govoplan-deploy.pyz.sha256
|
(cd runtime-output && sha256sum govoplan-deploy.pyz > govoplan-deploy.pyz.sha256)
|
||||||
sha256sum runtime-output/distribution-manifest.json > runtime-output/distribution-manifest.json.sha256
|
(cd runtime-output && sha256sum distribution-manifest.json > distribution-manifest.json.sha256)
|
||||||
rm runtime-output/signing-key.pem
|
rm runtime-output/signing-key.pem
|
||||||
- name: Verify the published bundle contract with the zipapp
|
- name: Verify the published bundle contract with the zipapp
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ inputs.version }}
|
VERSION: ${{ inputs.version }}
|
||||||
|
SIGNING_KEY_ID: ${{ secrets.RUNTIME_DISTRIBUTION_SIGNING_KEY_ID }}
|
||||||
run: |
|
run: |
|
||||||
|
(cd runtime-output && sha256sum --check govoplan-deploy.pyz.sha256)
|
||||||
|
(cd runtime-output && sha256sum --check distribution-manifest.json.sha256)
|
||||||
|
.runtime-build/bin/python - <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
keyring = json.loads(
|
||||||
|
Path("runtime-output/distribution-keyring.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
key_id = os.environ["SIGNING_KEY_ID"]
|
||||||
|
matches = [item for item in keyring["keys"] if item.get("key_id") == key_id]
|
||||||
|
if len(matches) != 1 or matches[0].get("status") != "active":
|
||||||
|
raise SystemExit("runtime signing key is not uniquely active in the keyring")
|
||||||
|
Path("runtime-output/runtime-release-public.pem").write_text(
|
||||||
|
matches[0]["public_key_pem"], encoding="utf-8"
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
openssl pkeyutl -verify -pubin \
|
||||||
|
-inkey runtime-output/runtime-release-public.pem -rawin \
|
||||||
|
-in runtime-output/govoplan-deploy.pyz \
|
||||||
|
-sigfile runtime-output/govoplan-deploy.pyz.sig
|
||||||
|
cp runtime-output/govoplan-deploy.pyz runtime-output/govoplan-deploy.tampered.pyz
|
||||||
|
printf '\0' >> runtime-output/govoplan-deploy.tampered.pyz
|
||||||
|
if openssl pkeyutl -verify -pubin \
|
||||||
|
-inkey runtime-output/runtime-release-public.pem -rawin \
|
||||||
|
-in runtime-output/govoplan-deploy.tampered.pyz \
|
||||||
|
-sigfile runtime-output/govoplan-deploy.pyz.sig >/dev/null 2>&1; then
|
||||||
|
echo "Tampered deployment bootstrap unexpectedly verified" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
MANIFEST_SHA256="$(cut -d' ' -f1 runtime-output/distribution-manifest.json.sha256)"
|
MANIFEST_SHA256="$(cut -d' ' -f1 runtime-output/distribution-manifest.json.sha256)"
|
||||||
python runtime-output/govoplan-deploy.pyz init \
|
python runtime-output/govoplan-deploy.pyz init \
|
||||||
--directory runtime-output/acceptance-install \
|
--directory runtime-output/acceptance-install \
|
||||||
@@ -236,14 +342,17 @@ jobs:
|
|||||||
python tools/checks/managed-ingress-drill.py
|
python tools/checks/managed-ingress-drill.py
|
||||||
--caddy-image "$MANAGED_INGRESS_IMAGE"
|
--caddy-image "$MANAGED_INGRESS_IMAGE"
|
||||||
--load-balancer-image "$LOAD_BALANCER_IMAGE"
|
--load-balancer-image "$LOAD_BALANCER_IMAGE"
|
||||||
|
--probe-image "$(jq -r '.platforms["linux/amd64"]' runtime-output/api-metadata.json)"
|
||||||
- name: Publish immutable Gitea release assets
|
- name: Publish immutable Gitea release assets
|
||||||
working-directory: govoplan
|
working-directory: govoplan
|
||||||
env:
|
env:
|
||||||
VERSION: ${{ inputs.version }}
|
VERSION: ${{ inputs.version }}
|
||||||
|
SOURCE_COMMIT: ${{ gitea.sha }}
|
||||||
GITEA_RELEASE_TOKEN: ${{ secrets.GOVOPLAN_RELEASE_TOKEN }}
|
GITEA_RELEASE_TOKEN: ${{ secrets.GOVOPLAN_RELEASE_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
python tools/release/publish-runtime-release.py \
|
python tools/release/publish-runtime-release.py \
|
||||||
--tag "v$VERSION" \
|
--tag "v$VERSION" \
|
||||||
|
--target-commit "$SOURCE_COMMIT" \
|
||||||
--title "GovOPlaN v$VERSION runtime distribution" \
|
--title "GovOPlaN v$VERSION runtime distribution" \
|
||||||
--asset runtime-output/govoplan-deploy.pyz \
|
--asset runtime-output/govoplan-deploy.pyz \
|
||||||
--asset runtime-output/govoplan-deploy.pyz.sig \
|
--asset runtime-output/govoplan-deploy.pyz.sig \
|
||||||
@@ -255,4 +364,6 @@ jobs:
|
|||||||
--asset runtime-output/evidence/api-sbom.cdx.json \
|
--asset runtime-output/evidence/api-sbom.cdx.json \
|
||||||
--asset runtime-output/evidence/web-sbom.cdx.json \
|
--asset runtime-output/evidence/web-sbom.cdx.json \
|
||||||
--asset runtime-output/evidence/api-provenance.json \
|
--asset runtime-output/evidence/api-provenance.json \
|
||||||
--asset runtime-output/evidence/web-provenance.json
|
--asset runtime-output/evidence/web-provenance.json \
|
||||||
|
--asset runtime-output/evidence/runtime-smoke-amd64.json \
|
||||||
|
--asset runtime-output/evidence/runtime-smoke-arm64.json
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
name: Runtime Ingress Drill
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
caddy_image:
|
||||||
|
description: Digest-pinned Caddy image
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
load_balancer_image:
|
||||||
|
description: Digest-pinned HAProxy image
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
probe_image:
|
||||||
|
description: Digest-pinned amd64 GovOPlaN API image
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
managed-ingress:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
|
with:
|
||||||
|
path: govoplan
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- name: Authenticate runtime image pull
|
||||||
|
env:
|
||||||
|
REGISTRY_USERNAME: ${{ secrets.GOVOPLAN_REGISTRY_USERNAME }}
|
||||||
|
REGISTRY_TOKEN: ${{ secrets.GOVOPLAN_REGISTRY_TOKEN }}
|
||||||
|
run: echo "$REGISTRY_TOKEN" | docker login git.add-ideas.de --username "$REGISTRY_USERNAME" --password-stdin
|
||||||
|
- name: Exercise the managed ingress boundary
|
||||||
|
working-directory: govoplan
|
||||||
|
env:
|
||||||
|
CADDY_IMAGE: ${{ inputs.caddy_image }}
|
||||||
|
LOAD_BALANCER_IMAGE: ${{ inputs.load_balancer_image }}
|
||||||
|
PROBE_IMAGE: ${{ inputs.probe_image }}
|
||||||
|
run: >-
|
||||||
|
python tools/checks/managed-ingress-drill.py
|
||||||
|
--caddy-image "$CADDY_IMAGE"
|
||||||
|
--load-balancer-image "$LOAD_BALANCER_IMAGE"
|
||||||
|
--probe-image "$PROBE_IMAGE"
|
||||||
@@ -145,7 +145,54 @@ installation. Separate amd64/arm64 API and WebUI images are joined into OCI
|
|||||||
indexes and run as non-root identities. The release assets include CycloneDX
|
indexes and run as non-root identities. The release assets include CycloneDX
|
||||||
application SBOMs, SLSA-style provenance, exact composition evidence, the
|
application SBOMs, SLSA-style provenance, exact composition evidence, the
|
||||||
single-file deployer, its detached Ed25519 signature, and a signed, expiring
|
single-file deployer, its detached Ed25519 signature, and a signed, expiring
|
||||||
distribution manifest.
|
distribution manifest. Evidence generation and signing run through the
|
||||||
|
workflow's isolated release Python environment so their cryptographic tooling
|
||||||
|
is explicit and independent of packages preinstalled in the Actions runner.
|
||||||
|
The API image points Core at the migration scripts installed from the verified
|
||||||
|
wheel under `/opt/govoplan/runtime/govoplan_core_runtime`; migrations therefore
|
||||||
|
do not depend on a source checkout or the build host's Python installation
|
||||||
|
scheme.
|
||||||
|
Before publication, the exact amd64 and arm64 image manifests each run release
|
||||||
|
migrations against the pinned PostgreSQL image, reach API and WebUI readiness
|
||||||
|
as non-root/read-only processes, and complete a task through the pinned Redis
|
||||||
|
image and packaged worker. Sanitized per-platform smoke receipts are retained
|
||||||
|
as immutable release assets.
|
||||||
|
PostgreSQL and Redis indexes are resolved to untagged platform-child digests
|
||||||
|
before each smoke run. This keeps the evidence architecture-specific and
|
||||||
|
avoids retargeting one local Docker tag between incompatible platforms.
|
||||||
|
The CI host registers arm64 execution with an explicitly supplied,
|
||||||
|
digest-pinned `tonistiigi/binfmt` image immediately before the smoke. This
|
||||||
|
privileged helper is confined to the release runner and is never part of a
|
||||||
|
GovOPlaN target deployment or its runtime image set.
|
||||||
|
Because QEMU user-mode execution triggers Redis's arm64 host-kernel COW guard,
|
||||||
|
the arm64 smoke suppresses only `ARM64-COW-BUG` while persistence, snapshots,
|
||||||
|
and append-only files are disabled. Target Redis services never inherit this
|
||||||
|
test-only option.
|
||||||
|
The smoke also proves a bounded post-migration table contract and aborts as
|
||||||
|
soon as a required container exits, rather than allowing a dead process to
|
||||||
|
consume the full readiness timeout.
|
||||||
|
|
||||||
|
Ingress acceptance streams generated configuration into Docker-managed
|
||||||
|
volumes before starting the read-only containers. It therefore also works when
|
||||||
|
an Actions job reaches a host or remote Docker daemon through a mounted socket;
|
||||||
|
the drill never assumes that a job-container path is visible to that daemon.
|
||||||
|
The drill allocates explicit loopback-only host ports and verifies Docker's
|
||||||
|
host binding configuration, avoiding daemon-specific random-port shorthand
|
||||||
|
behavior. Because an Actions job and deployment containers may be Docker
|
||||||
|
siblings, functional HTTP/TLS checks run from the digest-pinned API image on
|
||||||
|
the deployment network instead of assuming the Docker host is job-local.
|
||||||
|
The dispatch-only `Runtime Ingress Drill` workflow exposes the same bounded
|
||||||
|
check independently so ingress changes can be diagnosed before an immutable
|
||||||
|
runtime publication; it accepts only digest-pinned Caddy, HAProxy, and API
|
||||||
|
images and has no push trigger.
|
||||||
|
The official Caddy binary carries the `NET_BIND_SERVICE` file capability. The
|
||||||
|
managed-ingress container therefore drops every capability and adds back only
|
||||||
|
`NET_BIND_SERVICE`; otherwise Linux rejects the binary at `execve` before its
|
||||||
|
high-port configuration can start. `no-new-privileges`, a read-only root
|
||||||
|
filesystem, and non-privileged container ports remain enforced.
|
||||||
|
The bounded setup helper writes only generated public configuration as root so
|
||||||
|
it can initialize a new volume; the actual HAProxy process retains the image's
|
||||||
|
non-root identity and runs read-only with all capabilities dropped.
|
||||||
|
|
||||||
The manifest contract is
|
The manifest contract is
|
||||||
[`runtime-distribution-manifest.schema.json`](runtime-distribution-manifest.schema.json),
|
[`runtime-distribution-manifest.schema.json`](runtime-distribution-manifest.schema.json),
|
||||||
@@ -444,15 +491,27 @@ of the reviewed update recipe instead of a non-functional update button.
|
|||||||
|
|
||||||
## Distribution Workflow
|
## Distribution Workflow
|
||||||
|
|
||||||
The downloadable entry point is a release asset. Obtain the zipapp, detached
|
The downloadable entry point is a reproducible release asset: sorted source
|
||||||
signature, checksum, and trusted public keyring through independently
|
paths, fixed ZIP metadata, fixed compression settings, and identical source
|
||||||
authenticated paths before execution:
|
bytes produce an identical zipapp regardless of checkout timestamps. Obtain the
|
||||||
|
zipapp, detached signature, checksum, and trusted public keyring through
|
||||||
|
independently authenticated paths before execution:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
curl --proto '=https' --tlsv1.2 --fail --location \
|
curl --proto '=https' --tlsv1.2 --fail --location \
|
||||||
https://git.add-ideas.de/GovOPlaN/govoplan/releases/download/vX.Y.Z/govoplan-deploy.pyz \
|
https://git.add-ideas.de/GovOPlaN/govoplan/releases/download/vX.Y.Z/govoplan-deploy.pyz \
|
||||||
--output govoplan-deploy.pyz
|
--output govoplan-deploy.pyz
|
||||||
sha256sum --check govoplan-deploy.pyz.sha256
|
sha256sum --check govoplan-deploy.pyz.sha256
|
||||||
|
python3 - <<'PY'
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
keyring = json.loads(Path("distribution-keyring.json").read_text())
|
||||||
|
active = [key for key in keyring["keys"] if key["status"] == "active"]
|
||||||
|
if len(active) != 1:
|
||||||
|
raise SystemExit("expected exactly one active runtime release key")
|
||||||
|
Path("runtime-release-public.pem").write_text(active[0]["public_key_pem"])
|
||||||
|
PY
|
||||||
openssl pkeyutl -verify -pubin -inkey runtime-release-public.pem -rawin \
|
openssl pkeyutl -verify -pubin -inkey runtime-release-public.pem -rawin \
|
||||||
-in govoplan-deploy.pyz -sigfile govoplan-deploy.pyz.sig
|
-in govoplan-deploy.pyz -sigfile govoplan-deploy.pyz.sig
|
||||||
python3 govoplan-deploy.pyz init
|
python3 govoplan-deploy.pyz init
|
||||||
|
|||||||
@@ -15,7 +15,14 @@ machine-readable field, label, translation, route, API-reference, and module
|
|||||||
manifest evidence. This hand-maintained document remains the reviewed product
|
manifest evidence. This hand-maintained document remains the reviewed product
|
||||||
interpretation and rollout ledger; generated evidence does not replace it.
|
interpretation and rollout ledger; generated evidence does not replace it.
|
||||||
|
|
||||||
Snapshot refreshed: 2026-07-22.
|
Snapshot refreshed: 2026-08-03.
|
||||||
|
|
||||||
|
The generated snapshot contains 65 module manifests, 35 WebUI-contributing
|
||||||
|
repositories, 40 statically declared module routes, 1,156 UI fields, and 836
|
||||||
|
backend endpoints. All backend endpoints are classified and no stale endpoint
|
||||||
|
declarations were found. The 234 endpoints without a static WebUI reference are
|
||||||
|
kept visible as review evidence; they may intentionally serve workers, public
|
||||||
|
clients, connectors, or external integrations.
|
||||||
|
|
||||||
Evidence was read from tracked Git `HEAD` in the local GovOPlaN checkouts:
|
Evidence was read from tracked Git `HEAD` in the local GovOPlaN checkouts:
|
||||||
|
|
||||||
@@ -55,33 +62,50 @@ Inventory states:
|
|||||||
| Session/bootstrap state | `govoplan-core` `App.tsx` and `AppShell` | All browser sessions during bootstrap | Understand that session/platform state is loading; state contract | Unreviewed; core shell |
|
| Session/bootstrap state | `govoplan-core` `App.tsx` and `AppShell` | All browser sessions during bootstrap | Understand that session/platform state is loading; state contract | Unreviewed; core shell |
|
||||||
| `/` authenticated redirect | `govoplan-core` chooses the first visible navigation destination | Authenticated; result depends on visible nav contributions | Enter the actor's first accessible service area; navigation behavior, not a content page | Unreviewed; focused-view/default-route work must preserve this fallback |
|
| `/` authenticated redirect | `govoplan-core` chooses the first visible navigation destination | Authenticated; result depends on visible nav contributions | Enter the actor's first accessible service area; navigation behavior, not a content page | Unreviewed; focused-view/default-route work must preserve this fallback |
|
||||||
| `/dashboard` fallback | `govoplan-core` `DashboardPage` only when the Dashboard module is absent | Authenticated; no route-specific scope in core | Cross-module starting point; dashboard | Unreviewed; compare with module dashboard before shared changes |
|
| `/dashboard` fallback | `govoplan-core` `DashboardPage` only when the Dashboard module is absent | Authenticated; no route-specific scope in core | Cross-module starting point; dashboard | Unreviewed; compare with module dashboard before shared changes |
|
||||||
| `/settings` | `govoplan-core` `SettingsPage` | Authenticated; contributed sections and integrations filter internally | Profile, UI/workspace preference, local connection, and user-scoped integration settings; configuration | Unreviewed; Core #225 program |
|
| `/settings` | `govoplan-core` `SettingsPage` | Authenticated; contributed sections and integrations filter internally | Profile, UI/workspace preference, local connection, and user-scoped integration settings; configuration | Core-owned pattern migration complete in [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225), commit `fa32cca` |
|
||||||
| Shell chrome | `AppShell`, `Titlebar`, `IconRail`, `BreadcrumbBar`, `HelpMenu`, language menu, unsaved-change provider | Public/authenticated variants; nav filtered later | Tenant/actor context, global navigation, help, language, session and maintenance state | Unreviewed; platform-owned prerequisite for focused views |
|
| Shell chrome | `AppShell`, `Titlebar`, `IconRail`, `BreadcrumbBar`, `HelpMenu`, language menu, unsaved-change provider | Public/authenticated variants; nav filtered later | Tenant/actor context, global navigation, help, language, session and maintenance state | Unreviewed; platform-owned prerequisite for focused views |
|
||||||
|
|
||||||
## Direct Module Route Contributions
|
## Direct Module Route Contributions
|
||||||
|
|
||||||
The access guard column reports only the route-level declaration in
|
The access column summarizes only the route-level declaration in `module.ts`.
|
||||||
`module.ts`. Inner APIs and controls may impose additional checks.
|
Inner APIs and controls may impose additional checks. Public and compatibility
|
||||||
|
routes are called out explicitly because they do not have the same manifest
|
||||||
|
semantics as authenticated navigation routes.
|
||||||
|
|
||||||
| Route | Owner / render evidence | Route-level access evidence | Primary task | Target archetype | Status / priority |
|
| Routes | Owner | Route-level access | Primary archetype | Migration issue |
|
||||||
| --- | --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `/admin` | `govoplan-access` `AdminPage` | Any core `adminReadScopes` | Administer system and tenant concerns assembled from module sections | Administration/configuration | Contributed; unreviewed; P1 under [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225) |
|
| `/admin` | Access | Any declared administration/read scope | Administration/configuration host | Access pattern migration complete in [Access #19](https://git.add-ideas.de/GovOPlaN/govoplan-access/issues/19), commit `1409dbf`; shared host contract complete in [Core #225](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/225) |
|
||||||
| `/address-book` | `govoplan-addresses` `AddressBookPage` | `addresses:contact:read` | Browse and manage contacts, address books, and lists | Directory/list-detail | Contributed; unreviewed; P2 after Campaign |
|
| `/address-book` | Addresses | `addresses:contact:read` | Governed source directory, contact/list detail, external-provider operation, governance facts, and reversible correction | Addresses pattern migration complete in [Addresses #23](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/23), commit `f9a7185` |
|
||||||
| `/calendar` | `govoplan-calendar` `CalendarPage` | `calendar:event:read` | Browse calendars/events and act on calendar data | Directory/list-detail | Contributed; metadata gap; unreviewed; P2 after Campaign |
|
| `/approvals` | Approvals | `approvals:workspace:read` | Work queue/guided decision | Approvals pattern migration complete in [Approvals #3](https://git.add-ideas.de/GovOPlaN/govoplan-approvals/issues/3), commit `24e9559` |
|
||||||
| `/campaigns` | `govoplan-campaign` `CampaignListPage` | `campaigns:campaign:read` | Find, compare, create, and open campaigns | List-detail entry | Pilot; P1 [Campaign #74](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/74) |
|
| `/calendar` | Calendar | `calendar:event:read` | Full-height calendar workspace with filterable collection/agenda sidebar, continuous and bounded date views, guarded VEVENT and source editors, synchronized-source status, durable outbox recovery, and destructive remote-move evidence | Calendar pattern migration complete in [Calendar #22](https://git.add-ideas.de/GovOPlaN/govoplan-calendar/issues/22), commit `d7fd944` |
|
||||||
| `/campaigns/:campaignId/*` | `govoplan-campaign` `CampaignResourceRoute` and `CampaignWorkspace` | `campaigns:campaign:read`, plus resource probe | Configure, review, send, and inspect one campaign/version | List-detail workspace containing edit, review, monitoring, and evidence surfaces | Pilot; P1 Campaign #74 |
|
| `/campaigns`, `/campaigns/:campaignId/*`, `/campaigns/queue`, `/campaigns/reports`, `/templates` | Campaign | Campaign read/report/control scopes; template route has no route guard | List-detail, guided review, monitoring, reporting | [Campaign #74](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/74) |
|
||||||
| `/operator` | `govoplan-campaign` `OperatorQueuePage` | `campaigns:campaign:read` and any of queue, control, retry, or reconcile | Monitor and intervene in campaign jobs through authority-specific controls | Monitoring/work queue | Pilot; durable queue controls delivered in [Campaign #78](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/78); #74 audit remains |
|
| `/operator` | Campaign | Campaign read plus queue/control scope | Compatibility redirect to `/campaigns/queue` | Campaign #74; retire under the compatibility policy |
|
||||||
| `/reports` | `govoplan-campaign` `AggregateReportsPage` | `campaigns:report:read` | Compare privacy-protected cross-campaign outcome totals without recipient detail, diagnostics, export, or drill-down | Aggregate reporting | Pilot; aggregate-reader surface delivered in [Campaign #80](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/80); #74 audit remains |
|
| `/cases`, `/cases/:caseId` | Cases | `cases:case:read` | Governed case directory and detail workspace with guarded OCC lifecycle editor, provider-owned references, immutable timeline/history, and confirmed object-access editor | Cases pattern migration complete in [Cases #4](https://git.add-ideas.de/GovOPlaN/govoplan-cases/issues/4), commit `43b4cc8` |
|
||||||
| `/templates` | `govoplan-campaign` `TemplatesPage` | No route guard declared in `module.ts` | Browse/manage campaign templates | Directory/list-detail | Pilot audit; permission intent must be verified; P2 |
|
| `/committee` | Committee | `committee:workspace:read` | Governed workspace | Committee pattern migration complete in [Committee #2](https://git.add-ideas.de/GovOPlaN/govoplan-committee/issues/2), commit `e64af30` |
|
||||||
| `/dashboard` | `govoplan-dashboard` `DashboardPage` | No route-specific scope | Assemble module-provided actionable widgets | Dashboard | Contributed; unreviewed; P2 |
|
| `/dashboard` | Dashboard | No route-specific scope | View-specific personal workspace with module/permission-filtered widget library, guarded four-column composition, nested widget settings, server/browser fallback, and optimistic layout persistence | Dashboard pattern migration complete in [Dashboard #3](https://git.add-ideas.de/GovOPlaN/govoplan-dashboard/issues/3), commit `da3947f` |
|
||||||
| `/docs` | `govoplan-docs` `DocsPage` | Docs read or system/tenant settings read scopes | Read configured, available, and evidence-aware documentation | Documentation directory/reference | Contributed; unreviewed; P1 [Docs #15](https://git.add-ideas.de/GovOPlaN/govoplan-docs/issues/15) after initial pattern content |
|
| `/dataflow` | Dataflow | Pipeline read/admin | Governed library, guarded graph/constrained-SQL definition editor, typed node inspector, bounded intermediate preview, automation triggers, and durable run/deployment evidence | Dataflow pattern migration complete in [Dataflow #20](https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/20), commit `109ddcd` |
|
||||||
| `/files` | `govoplan-files` `FilesPage` | `files:file:read` | Browse folders/files and perform managed-file work | Directory/explorer | Contributed; metadata gap; unreviewed; P2 after Campaign |
|
| `/datasources` | Datasources | Catalogue read/source admin | Governed catalogue, staging preflight, optional-origin directory, authority editor, and immutable evidence | Datasources pattern migration complete in [Datasources #7](https://git.add-ideas.de/GovOPlaN/govoplan-datasources/issues/7), commit `6406ce7` |
|
||||||
| `/idm` | `govoplan-idm` `IdmPage` | Any IDM assignment/write or organization function-assign scope | Inspect and govern identity/function assignments | List-detail/configuration | Contributed; unreviewed; P2 |
|
| `/distribution-lists` | Distribution Lists | List read/write/admin | Governed directory, immutable-revision editor, expansion preview, and evidence register | Distribution Lists pattern migration complete in [Distribution Lists #8](https://git.add-ideas.de/GovOPlaN/govoplan-dist-lists/issues/8), commit `6cdd804` |
|
||||||
| `/mail` | `govoplan-mail` `MailboxPage` | `mail:mailbox:read` | Browse mailboxes and messages | Directory/list-detail | Contributed; metadata gap; unreviewed; P2 after Campaign |
|
| `/docs` | Docs | Documentation or settings read | Documentation/reference | [Docs #15](https://git.add-ideas.de/GovOPlaN/govoplan-docs/issues/15) |
|
||||||
| `/notifications` | `govoplan-notifications` `NotificationCenterPage` | `notifications:notification:read` | Inspect and acknowledge notification state | List-detail/inbox | Contributed without a nav item or backend frontend metadata; navigation intent unknown; P2 discovery |
|
| `/files` | Files | `files:file:read` | Directory/explorer | Files pattern migration complete in [Files #42](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/42), commit `d8ae506` |
|
||||||
| `/ops` | `govoplan-ops` `OpsPage` | Ops read or system/tenant settings read scopes | Inspect runtime health and readiness | Monitoring | Contributed; unreviewed; P2 |
|
| `/forms` | Forms | `forms:definition:read` | Definition library/editor | Forms pattern migration complete in [Forms #4](https://git.add-ideas.de/GovOPlaN/govoplan-forms/issues/4), commit `e505536` |
|
||||||
| `/organizations` | `govoplan-organizations` `OrganizationsPage` | Organization model/unit/function or admin settings read scopes | Model and inspect organizational structures/functions | Directory/list-detail | Contributed; unreviewed; P2 |
|
| `/forms-runtime`, `/forms-runtime/:instanceId` | Forms Runtime | Participate or workspace read | Guided form execution | Forms Runtime pattern migration complete in [Forms Runtime #5](https://git.add-ideas.de/GovOPlaN/govoplan-forms-runtime/issues/5), commit `07dd35b` |
|
||||||
| `/scheduling` | `govoplan-scheduling` `SchedulingPage` | `scheduling:schedule:read` | Plan and decide scheduling requests and availability | List-detail/guided decision | Contributed; metadata gap; unreviewed; P2 |
|
| `/idm` | IDM | Assignment, function-change, relationship, or organization scopes | Directory/governed change | IDM pattern migration complete in [IDM #12](https://git.add-ideas.de/GovOPlaN/govoplan-idm/issues/12), commit `d864317` |
|
||||||
|
| `/mail`, `/mail/bounces` | Mail | Mailbox or bounce read/manage | Directory/explorer, operational evidence | Mail pattern migration complete in [Mail #20](https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/20), commit `7844d9c` |
|
||||||
|
| `/notifications` | Notifications | `notifications:notification:read` | Inbox/list-detail with guarded recipient state, confirmed local cancellation/dispatch, and sanitized delivery evidence | Notifications pattern migration complete in [Notifications #4](https://git.add-ideas.de/GovOPlaN/govoplan-notifications/issues/4), commit `ad6a31f` |
|
||||||
|
| `/ops` | Ops | Operations or settings read | Monitoring/evidence with contextual run, drain, readiness-blocker, and recovery guidance | Ops pattern migration complete in [Ops #4](https://git.add-ideas.de/GovOPlaN/govoplan-ops/issues/4), commit `2b32643` |
|
||||||
|
| `/organizations` | Organizations | Model/unit/function or settings read | Directory/hierarchy editor | Organizations pattern migration complete in [Organizations #7](https://git.add-ideas.de/GovOPlaN/govoplan-organizations/issues/7), commit `97acfcb` |
|
||||||
|
| `/portal` | Portal | `portal:service:read` | Service portal | [Portal #2](https://git.add-ideas.de/GovOPlaN/govoplan-portal/issues/2) |
|
||||||
|
| `/postbox` | Postbox | `postbox:postbox:read` | Inbox/list-detail | Postbox pattern migration complete in [Postbox #26](https://git.add-ideas.de/GovOPlaN/govoplan-postbox/issues/26), commit `a97eb3b` |
|
||||||
|
| `/projects` | Projects | `projects:project:read` | List-detail/project workspace | [Projects #2](https://git.add-ideas.de/GovOPlaN/govoplan-projects/issues/2) |
|
||||||
|
| `/reporting`, `/reports` | Reporting | `reporting:definition:read` | Reporting/definition library | [Reporting #8](https://git.add-ideas.de/GovOPlaN/govoplan-reporting/issues/8) |
|
||||||
|
| `/risk-compliance` | Risk Compliance | Workspace or sanctions read | Immutable source evidence, version-pinned screening, list-detail review, and revisioned assurance graph with explicit blockers and consequences | Risk Compliance pattern migration complete in [Risk Compliance #8](https://git.add-ideas.de/GovOPlaN/govoplan-risk-compliance/issues/8), commit `24d80a6` |
|
||||||
|
| `/scheduling` | Scheduling | `scheduling:schedule:read` | List-detail/guided decision | Scheduling pattern migration complete in [Scheduling #8](https://git.add-ideas.de/GovOPlaN/govoplan-scheduling/issues/8), commit `c17cbda` |
|
||||||
|
| `/scheduling/public/:requestId/:token` | Scheduling | Public signed token | Public participation | Scheduling #8 complete in `c17cbda` |
|
||||||
|
| `/search` | Search | `search:result:read` | Search overlay/results | [Search #4](https://git.add-ideas.de/GovOPlaN/govoplan-search/issues/4) |
|
||||||
|
| `/templates` | Templates | Template read/write/publish/render/admin | Governed library, immutable-revision editor, compatibility preview, and render evidence | Templates pattern migration complete in [Templates #5](https://git.add-ideas.de/GovOPlaN/govoplan-templates/issues/5), commit `72fafa2` |
|
||||||
|
| `/voting` | Voting | `voting:ballot:read` | Governed ballot workspace | Voting pattern migration complete in [Voting #1](https://git.add-ideas.de/GovOPlaN/govoplan-voting/issues/1), commit `2625990` |
|
||||||
|
| `/workflow` | Workflow | Definition read or instance admin | Graph editor/execution evidence | [Workflow #15](https://git.add-ideas.de/GovOPlaN/govoplan-workflow/issues/15) |
|
||||||
|
|
||||||
## Manifest And Runtime Route Alignment
|
## Manifest And Runtime Route Alignment
|
||||||
|
|
||||||
@@ -91,28 +115,27 @@ loading reason about the configured interface without executing module UI code.
|
|||||||
is recorded here as an evidence gap; this inventory does not infer whether each
|
is recorded here as an evidence gap; this inventory does not infer whether each
|
||||||
gap is intentional.
|
gap is intentional.
|
||||||
|
|
||||||
| Module | `module.ts` routes | Backend manifest frontend routes | Backend nav alignment | Result |
|
The generated comparison is aligned for all authenticated canonical routes.
|
||||||
| --- | --- | --- | --- | --- |
|
Two deliberate exceptions remain visible:
|
||||||
| Access | `/admin` | `/admin` | Aligned | Described |
|
|
||||||
| Addresses | `/address-book` | `/address-book` | Aligned | Described |
|
|
||||||
| Admin | No direct route; `admin.sections` | None | Not applicable | Composed surface |
|
|
||||||
| Audit | No direct route; `admin.sections` | None | Not applicable | Composed surface |
|
|
||||||
| Calendar | `/calendar` | None | `/calendar` nav exists | Metadata gap |
|
|
||||||
| Campaign | Five routes | None | Four top-level nav items exist | Metadata gap; wildcard resource route is also undescribed |
|
|
||||||
| Dashboard | `/dashboard` | `/dashboard` | Aligned | Described |
|
|
||||||
| Docs | `/docs` | `/docs` | Aligned | Described |
|
|
||||||
| Files | `/files` | None | `/files` nav exists | Metadata gap |
|
|
||||||
| IDM | `/idm` | `/idm` | Aligned | Described |
|
|
||||||
| Mail | `/mail` | None | `/mail` nav exists | Metadata gap |
|
|
||||||
| Notifications | `/notifications` | No frontend metadata | No nav item | Metadata and discovery gap |
|
|
||||||
| Ops | `/ops` | `/ops` | Aligned | Described |
|
|
||||||
| Organizations | `/organizations` | `/organizations` | Aligned | Described |
|
|
||||||
| Policy | No direct route; `admin.sections` | None | Not applicable | Composed surface |
|
|
||||||
| Scheduling | `/scheduling` | None | `/scheduling` nav exists | Metadata gap |
|
|
||||||
|
|
||||||
Before a release claims a complete configured-system route inventory, add a
|
- Campaign contributes `/operator` as a compatibility redirect for saved View
|
||||||
contract check or explicit exceptions so executable routes and manifest
|
projections; its canonical and manifest-declared destination is
|
||||||
metadata cannot silently diverge.
|
`/campaigns/queue`.
|
||||||
|
- Scheduling contributes `/scheduling/public/:requestId/:token` through the
|
||||||
|
separate `publicRoutes` contract. Authenticated manifest routes intentionally
|
||||||
|
do not describe public signed-token entry points yet.
|
||||||
|
|
||||||
|
Admin, Audit, Policy, Tenancy, and Views contribute composed administration or
|
||||||
|
settings surfaces rather than direct routes. Their migration issues are
|
||||||
|
[Admin #8](https://git.add-ideas.de/GovOPlaN/govoplan-admin/issues/8),
|
||||||
|
[Audit #8](https://git.add-ideas.de/GovOPlaN/govoplan-audit/issues/8),
|
||||||
|
[Policy #11](https://git.add-ideas.de/GovOPlaN/govoplan-policy/issues/11),
|
||||||
|
[Tenancy #6](https://git.add-ideas.de/GovOPlaN/govoplan-tenancy/issues/6), and
|
||||||
|
[Views #2](https://git.add-ideas.de/GovOPlaN/govoplan-views/issues/2).
|
||||||
|
|
||||||
|
Release evidence must continue to run the generated inventory and manifest
|
||||||
|
shape checks so new executable routes, public routes, aliases, and composed
|
||||||
|
surfaces cannot silently diverge from their declared metadata.
|
||||||
|
|
||||||
## Composed Surfaces And Extension Points
|
## Composed Surfaces And Extension Points
|
||||||
|
|
||||||
@@ -121,25 +144,108 @@ enabled and the actor passes the declared filters.
|
|||||||
|
|
||||||
| Host surface | Contributor and evidence | Contributed regions/actions | Pattern implication | Audit |
|
| Host surface | Contributor and evidence | Contributed regions/actions | Pattern implication | Audit |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `/admin` | Access host (`AdminPage`) | System tenants/users/roles, tenant users/groups/roles/API keys/settings, function-role mappings, user/group mail and file connector scopes | One stable admin information architecture must contain both host-owned and contributed sections | Unreviewed; P1 Core #225 |
|
| `/admin` | Access host (`AdminPage`) | System tenants/users/roles, tenant users/groups/roles/API keys/settings, function-role mappings, user/group mail and file connector scopes | One stable admin information architecture must contain both host-owned and contributed sections | Pattern migration, contextual help, explained permission/protection states, optional-module blockers, localization, and focused evidence complete in Access #19 (`1409dbf`); Core #225 shared host contract complete |
|
||||||
| `/admin` | `govoplan-admin` `admin.sections` | Overview; system settings; configuration changes; configuration packages; role/group templates; module management | Configuration, guided operations, review/preflight, consequence | In progress under Core #225; surface-level evidence still needed |
|
| `/admin` | `govoplan-admin` `admin.sections` | Overview; system settings; configuration changes; configuration packages; role/group templates; module management | Configuration, guided operations, review/preflight, consequence | Pattern migration, contextual help, explained permission/protection/applicability states, guarded consequential actions, localization, and focused evidence complete in Admin #8 (`d428f33`) |
|
||||||
| `/admin` | `govoplan-audit` `admin.sections` | System audit; tenant audit | Evidence/provenance and reporting | Unreviewed |
|
| `/admin` | `govoplan-tenancy` `admin.sections` | System tenant registry and active-tenant settings | Administration directory, effective configuration, lifecycle consequence | Pattern migration, contextual help, explained permission/lifecycle/system-policy states, dirty-state guards, localization, and focused evidence complete in Tenancy #6 (`e76fe16`) |
|
||||||
| `/admin` | `govoplan-files` `admin.sections` and `files.connectors` | System and tenant file connections plus scoped connector managers used by Access | Adaptive configuration, discovery/test, policy and credentials | First migration family in Core #225; verification incomplete in this inventory |
|
| `/admin` | `govoplan-audit` `admin.sections` | System audit; tenant audit | Evidence/provenance and reporting | Pattern migration, localized evidence projection, contextual help, and focused tests complete in Audit #8 (`6d3fcc1`) |
|
||||||
| `/admin` | `govoplan-organizations` `admin.sections` | Tenant organization settings | Configuration/list-detail | Unreviewed |
|
| `/admin` | `govoplan-files` `admin.sections` and `files.connectors` | System and tenant file connections plus scoped connector managers used by Access | Adaptive configuration, discovery/test, policy and credentials | Pattern migration, contextual help, blocker explanations and focused evidence complete in Files #42 (`d8ae506`) |
|
||||||
| `/admin` | `govoplan-policy` `admin.sections` | System, tenant, group, and user retention | Effective value, source/provenance, consequential configuration | Unreviewed; Core #225 phase 4 |
|
| `/admin` | `govoplan-organizations` `admin.sections` | Tenant organization settings | Configuration/list-detail | Pattern migration, tenant-owned provenance, contextual help, guarded settings/editor drafts, explained permission states, localization and focused evidence complete in Organizations #7 (`97acfcb`) |
|
||||||
| `/admin` and `/settings` | `govoplan-mail` `mail.profiles` | System/tenant/group/user mail profile and policy managers | Same server/credential/policy grammar as file connectors | Unreviewed; Core #225 mail migration |
|
| `/admin` | `govoplan-policy` `admin.sections` | System, tenant, group, and user retention | Effective value, source/provenance, consequential configuration | Pattern migration complete in Policy #11 (`f964ed7`) with Core editor contract `fa32cca` |
|
||||||
| `/settings` | Core host | Profile; interface; workspace; local connection | Personal configuration with adaptive forms and immediate feedback | Unreviewed |
|
| `/admin` and `/settings` | `govoplan-mail` `mail.profiles` | System/tenant/group/user mail profile and policy managers | Same server/credential/policy grammar as file connectors | Pattern migration, contextual help, policy/target/permission blockers and focused evidence complete in Mail #20 (`7844d9c`; shared test-reason contract Core `2d0551a`) |
|
||||||
| `/settings` | Files and Mail named capabilities | User-scoped file connections and mail profiles/policy | Optional integration regions disappear cleanly when capability absent | Unreviewed |
|
| `/settings` | Core host | Profile; interface; workspace; local connection | Personal configuration with adaptive forms and immediate feedback | Pattern migration complete in Core #225 (`fa32cca`) |
|
||||||
|
| `/settings` | Files and Mail named capabilities | User-scoped file connections and mail profiles/policy | Optional integration regions disappear cleanly when capability absent | Files #42, Mail #20 and Core #225 complete |
|
||||||
|
| `/admin` and `/settings` | `govoplan-views` `admin.sections`, `settings.sections`, and `views.runtime` | System/tenant definition and assignment editors, personal/group editors, global selector | Versioned presentation projection with inheritance, lockout safeguards, optional directory targets, and no authorization effect | Pattern migration, contextual help, localized selector/editor, guarded drafts, explained inherited/permission/capability states, and focused evidence complete in Views #2 (`c125f33`) |
|
||||||
| `/settings` | `govoplan-notifications` `settings.sections` | Notification preferences | Personal configuration | Unreviewed |
|
| `/settings` | `govoplan-notifications` `settings.sections` | Notification preferences | Personal configuration | Unreviewed |
|
||||||
| `/dashboard` | Dashboard host and `dashboard.widgets` | Installed-modules widget; Ops health widget when Ops contributes it | Widget ordering, staleness, permissions, destination behavior | Unreviewed |
|
| `/dashboard` | Dashboard host and `dashboard.widgets` | Installed-modules widget; Ops health widget when Ops contributes it | Widget ordering, staleness, permissions, destination behavior | Unreviewed |
|
||||||
| `/organizations` | IDM `organizations.functionActions` | Action leading to assignment view filtered by IDM scopes | Cross-module context action through explicit capability | Unreviewed |
|
| `/organizations` | IDM `organizations.functionActions` | Action leading to assignment view filtered by IDM scopes | Cross-module context action through explicit capability | IDM pattern migration complete in IDM #12 (`d864317`) |
|
||||||
| Campaign attachments/import | Files `files.fileExplorer` | Folder tree, managed chooser, file listing/pattern resolution/sharing | Optional domain composition without sibling-private imports | Pilot audit under Campaign #74 |
|
| Campaign attachments/import | Files `files.fileExplorer` | Folder tree, managed chooser, file listing/pattern resolution/sharing | Optional domain composition without sibling-private imports | Pilot audit under Campaign #74 |
|
||||||
| Campaign review/send | Mail runtime `mail.devMailbox` | Mock-mail verification when backend advertises runtime capability | Optional review stage with unavailable/optional states | Pilot audit under Campaign #63/#62 |
|
| Campaign review/send | Mail runtime `mail.devMailbox` | Mock-mail verification when backend advertises runtime capability | Optional review stage with unavailable/optional states | Explicit intervention and review-progress vocabulary delivered in Campaign #63; send modes/progress delivered in #62/#79 |
|
||||||
|
|
||||||
Other named capability exports (`files.connectors`, `organizations.functionPicker`,
|
Other named capability exports (`files.connectors`, `organizations.functionPicker`,
|
||||||
and mail profile validation) are contracts consumed inside the composed surfaces
|
and mail profile validation) are contracts consumed inside the composed surfaces
|
||||||
above; they are not independent routes.
|
above; they are not independent routes.
|
||||||
|
|
||||||
|
## Core Configuration Surface Map
|
||||||
|
|
||||||
|
Core #225 now supplies and verifies the platform-owned configuration contract.
|
||||||
|
The durable Core inventory is
|
||||||
|
`govoplan-core/docs/INTERFACE_PATTERN_MIGRATION.md`.
|
||||||
|
|
||||||
|
| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `/settings` (`SettingsPage`) | Change personal profile, interface/workspace preferences, or local development connection | Two-zone typed settings workspace | Changes are user-scoped; save and test actions distinguish clean, busy, and active states | Contextual help, unsaved guard, typed controls and keyboard-explainable disabled actions in Core `fa32cca` |
|
||||||
|
| Reusable credentials (`CredentialEnvelopeManager`) | Compare and configure scoped reusable authentication material | Repeated administration plus adaptive create/edit | Secret values are write-only; permission and missing-owner states block mutation explicitly; deletion can break dependent connections | Actionable blocker, stable row actions, typed references, unsaved guard and shared destructive confirmation |
|
||||||
|
| Retention (`RetentionPolicyManagement`) | Inspect effective retention and narrow permitted local values | Effective-policy editor | Parent locks, source paths and write authority control whether sensitive evidence can be retained | Typed narrowing controls, source-path help, lock/target/permission blockers and clean/loading/save reasons |
|
||||||
|
| Shared configuration primitives | Compose module-owned settings without sibling-private imports | Platform behavior contract | Consequence, focus, help, async, confirmation and permission semantics remain consistent | Core component suites, 121 module-system tests and full-product type/build/bundle gates |
|
||||||
|
|
||||||
|
No primary Core configuration flow requires raw JSON. Expert JSON remains
|
||||||
|
limited to diagnostics, interchange, conflict evidence, or read-only inspection.
|
||||||
|
|
||||||
|
## Policy Surface Map
|
||||||
|
|
||||||
|
Policy #11 verifies the four composed retention sections. The durable
|
||||||
|
module-level inventory is
|
||||||
|
`govoplan-policy/docs/INTERFACE_PATTERN_MIGRATION.md`.
|
||||||
|
|
||||||
|
| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| System retention | Set the instance ceiling and run retention | Effective-policy editor plus destructive operation | An applied run can irreversibly redact/delete retained content; dry-run and applied evidence remain distinct | Core source-path/lock contract, permission and busy reasons, shared confirmation, typed/filterable outcome grid and audit-oriented wording |
|
||||||
|
| Tenant retention | Narrow the inherited system ceiling | Effective-policy editor | Tenant policy cannot silently loosen its parent | Core typed controls, effective path and parent-lock explanation |
|
||||||
|
| Group and user retention | Select an authorized target and narrow inherited policy | Targeted effective-policy editor | Selection exposes only bounded account/group labels; no retained content is returned | Delta-backed target loading, retry, missing-target blocker and responsive shared admin composition |
|
||||||
|
|
||||||
|
Automated evidence for Policy `f964ed7` comprises 50 backend/manifest tests,
|
||||||
|
the Policy interface structural gate, 65 manifest-shape checks, and the
|
||||||
|
full-product TypeScript/Vite build with structural localization, theme and
|
||||||
|
bundle-budget gates. Policy uses no sibling-private imports.
|
||||||
|
|
||||||
|
## Files Surface Map
|
||||||
|
|
||||||
|
Files #42 classifies and verifies the complete Files-owned route and composition
|
||||||
|
boundary. The durable module-level inventory is
|
||||||
|
`govoplan-files/docs/INTERFACE_PATTERN_MIGRATION.md`.
|
||||||
|
|
||||||
|
| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `/files` (`FilesPage`) | Browse spaces/folders and repeatedly act on current content | Full-height directory/explorer | Navigation is low consequence; upload, synchronize, move, copy and share are medium; delete is high | Stable two-pane composition, contextual help, selection/permission/state-specific disabled reasons, shared confirmation and responsive collapse |
|
||||||
|
| Upload/archive, transfer, rename and connector-import dialogs | Supply, validate and review one bounded change | Adaptive create/edit or guided import | Writes managed content and may resolve conflicts or import untrusted bytes | Shared dialogs/drop zone, bounded archive preflight, conflict review, explicit confirmation and no browser-native confirmation |
|
||||||
|
| Share/access explanation | Inspect or change who can use a resource | Review/decision | Grants can disclose content; delete/revoke changes access | Shared access explanation, action components and destructive confirmation; backend redaction remains authoritative |
|
||||||
|
| File connector tree and connection/credential dialogs | Compare and configure external endpoints and reusable credentials | Administration plus adaptive create/edit | Endpoint, secret and capability changes can enable remote access | Shared connection tree/forms/advanced panel, endpoint discovery and login test, unsaved-change guard, read-only deployment provenance and actionable disabled reasons |
|
||||||
|
| Connector policy card | Narrow effective connector use | Effective-policy editor | Inherited deny/allow rules affect lower scopes | Typed selectors, deny-precedence warning, effective sources, contextual admin help and permission blocker |
|
||||||
|
| `files.widget.spaces` | See available spaces and enter Files | Dashboard widget | Space/provider names remain permission-filtered | Shared loading, alert and status components; bounded configuration and refresh |
|
||||||
|
| `files.fileExplorer` capability | Select a governed managed snapshot for another module | Directory chooser | Exact file/version becomes another module's governed input | Capability-only composition, no sibling-private import, stable chooser/confirmation and exact snapshot evidence |
|
||||||
|
|
||||||
|
Automated evidence for commit `d8ae506` comprises 104 Files backend tests,
|
||||||
|
three focused Files WebUI structure tests, the full-product TypeScript/Vite
|
||||||
|
build, structural localization audit, theme contract and bundle budget. Shared
|
||||||
|
Dialog and disabled-tooltip behavior provide focus entry/return and
|
||||||
|
keyboard-reachable explanations; responsive source order is guarded at 1050 px
|
||||||
|
and 760 px. Secrets are not returned to the WebUI, and JSON remains only an
|
||||||
|
advanced provider-compatibility escape hatch rather than the primary editor.
|
||||||
|
|
||||||
|
## Mail Surface Map
|
||||||
|
|
||||||
|
Mail #20 classifies and verifies the complete Mail-owned route and composition
|
||||||
|
boundary. The durable module-level inventory is
|
||||||
|
`govoplan-mail/docs/INTERFACE_PATTERN_MIGRATION.md`.
|
||||||
|
|
||||||
|
| Surface / code evidence | Primary task | Target pattern | Material consequence/state | Completion evidence |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `/mail` (`MailboxPage`) | Browse an authorized provider mailbox without changing it | Full-height directory/explorer | Message metadata and content are private; every provider read is bounded and non-mutating | Stable three-pane composition, contextual help, explicit no-profile blocker, refresh reasons, keyboard rows, paging and responsive collapse |
|
||||||
|
| Mail profile tree and profile/server/credential dialogs | Compare and configure reusable transport identities | Administration plus guided/adaptive create/edit | Endpoint and credential changes can enable external effects | Shared connection tree/dialog/stage rail/forms, focused hierarchy editors, unsaved guard, connection tests, permission/target blockers and disabled-save reasons |
|
||||||
|
| Mail policy card | Narrow profile visibility, lower-scope definitions and transport/address patterns | Effective-policy editor | Inherited allow/deny rules affect delivery and lower scopes | Typed selectors and controls, effective source path, lock/read-only blocker, dirty-save state and contextual admin help |
|
||||||
|
| `/mail/bounces` watcher table | Configure and explicitly scan bounded IMAP evidence sources | Operational administration | Provider access changes durable source cursors and evidence | Shared grid/status/loading/alerts, actionable no-profile and busy states, field help and stable row actions |
|
||||||
|
| `/mail/bounces` observations and watcher removal | Review sanitized delivery outcomes or stop future scans | Evidence/reporting plus destructive confirmation | Recipient diagnostics are sensitive; watcher removal retains existing evidence | Bounded sanitized rows and shared confirmation with retained-evidence consequence |
|
||||||
|
| `mail.profiles` and reference-selector capabilities | Select/validate Mail-owned transport from another module | Governed capability composition | A selected identity can perform external effects | Stable references, Mail-owned authorization/secret resolution, no sibling-private imports and clean optional absence |
|
||||||
|
|
||||||
|
Automated evidence for Mail commit `7844d9c` and Core commit `2d0551a`
|
||||||
|
comprises 114 Mail backend tests, Mail's focused UI/model/structure suite, the
|
||||||
|
Core shared mail-component suite, 65 manifest-shape checks and the full-product
|
||||||
|
TypeScript/Vite build with structural localization, theme and bundle-budget
|
||||||
|
gates. Shared Dialog and disabled-tooltip behavior provides focus containment,
|
||||||
|
return and keyboard-reachable explanations. Responsive source order is guarded
|
||||||
|
at 1250 px, 900 px and 760 px. Passwords remain write-only, mailbox responses
|
||||||
|
are bounded, and bounce evidence excludes raw provider messages.
|
||||||
|
|
||||||
## Campaign Pilot Surface Map
|
## Campaign Pilot Surface Map
|
||||||
|
|
||||||
Campaign is detailed first because it exercises almost every archetype. The
|
Campaign is detailed first because it exercises almost every archetype. The
|
||||||
@@ -166,7 +272,7 @@ prove that the composition or states satisfy the pattern.
|
|||||||
| Campaign settings (`GlobalSettingsPage` settings view) | Configure campaign behavior | Adaptive configuration | Can alter validation/build/send behavior | #74 audit |
|
| Campaign settings (`GlobalSettingsPage` settings view) | Configure campaign behavior | Adaptive configuration | Can alter validation/build/send behavior | #74 audit |
|
||||||
| Mail policy (`MailSettingsPage` policy view) | Inspect/override effective mail policy | Effective policy/provenance editor | Inheritance and locks affect allowed delivery | #74; Core #225 policy pattern |
|
| Mail policy (`MailSettingsPage` policy view) | Inspect/override effective mail policy | Effective policy/provenance editor | Inheritance and locks affect allowed delivery | #74; Core #225 policy pattern |
|
||||||
| Campaign policy (`GlobalSettingsPage` policy view) | Inspect/override campaign policy | Effective policy/provenance editor | Inheritance, actor authority, and blocked edits | #74; Core #225 policy pattern |
|
| Campaign policy (`GlobalSettingsPage` policy view) | Inspect/override campaign policy | Effective policy/provenance editor | Inheritance, actor authority, and blocked edits | #74; Core #225 policy pattern |
|
||||||
| Review/send (`ReviewSendPage`) | Validate, build, mock-test, confirm/send, inspect results | Guided review/decision plus durable progress | External communication, bounded synchronous execution, persisted queue mode, partial effects, retries, evidence | Bounded synchronous and explicit/persisted queued modes delivered in [#62](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/62) and [#79](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/79); [#63](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/63) wording and #74 audit remain |
|
| Review/send (`ReviewSendPage`) | Validate, build, mock-test, confirm/send, inspect results | Guided review/decision plus durable progress | External communication, bounded synchronous execution, persisted queue mode, partial effects, retries, evidence | Blocking/non-blocking interventions and reviewed/remaining evidence delivered in [#63](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/63); bounded synchronous and explicit/persisted queued modes delivered in [#62](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/62) and [#79](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/79); #74 audit remains |
|
||||||
| Message and attachment detail overlays | Inspect one built/mock message and its attachment links | Stable detail/review dialog | Personal data, exact outbound content, reviewed state | Delivered and verified in [#59](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/59) and [#73](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/73) |
|
| Message and attachment detail overlays | Inspect one built/mock message and its attachment links | Stable detail/review dialog | Personal data, exact outbound content, reviewed state | Delivered and verified in [#59](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/59) and [#73](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/73) |
|
||||||
| Campaign report (`CampaignReportPage`) | Filter and inspect delivery outcomes | Reporting/list-detail | Partial, failed, explicitly excluded/skipped, SMTP/IMAP outcomes and retries | Server-owned filtering and counts delivered in [#65](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/65) with the full-result DataGrid contract from [Core #263](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/263); excluded semantics in [#66](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/66) |
|
| Campaign report (`CampaignReportPage`) | Filter and inspect delivery outcomes | Reporting/list-detail | Partial, failed, explicitly excluded/skipped, SMTP/IMAP outcomes and retries | Server-owned filtering and counts delivered in [#65](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/65) with the full-result DataGrid contract from [Core #263](https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/263); excluded semantics in [#66](https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/66) |
|
||||||
| Audit (`CampaignAuditPage`) | Inspect campaign evidence/history | Provenance timeline/report | Actor/action/effect trace | #74 audit |
|
| Audit (`CampaignAuditPage`) | Inspect campaign evidence/history | Provenance timeline/report | Actor/action/effect trace | #74 audit |
|
||||||
@@ -182,24 +288,26 @@ The five review stages currently named in code are `Validate and inspect`,
|
|||||||
results`. Campaign #63 owns the intervention and status vocabulary; Workflow is
|
results`. Campaign #63 owns the intervention and status vocabulary; Workflow is
|
||||||
not required to define or implement it.
|
not required to define or implement it.
|
||||||
|
|
||||||
## Repositories Without A WebUI Route Contribution
|
## Repositories Without A WebUI Package
|
||||||
|
|
||||||
The following local repositories contain a backend manifest but no
|
The generated manifest snapshot reports no WebUI package for:
|
||||||
`webui/src/module.ts` at this snapshot:
|
|
||||||
|
|
||||||
`govoplan-approvals`, `govoplan-assets`, `govoplan-booking`,
|
`govoplan-assets`, `govoplan-booking`, `govoplan-certificates`,
|
||||||
`govoplan-certificates`, `govoplan-committee`, `govoplan-consultation`,
|
`govoplan-connectors`, `govoplan-consultation`, `govoplan-contracts`,
|
||||||
`govoplan-contracts`, `govoplan-dist-lists`, `govoplan-evaluation`,
|
`govoplan-decisions`, `govoplan-encryption`, `govoplan-evaluation`,
|
||||||
`govoplan-facilities`, `govoplan-forms-runtime`, `govoplan-grants`,
|
`govoplan-facilities`, `govoplan-grants`, `govoplan-helpdesk`,
|
||||||
`govoplan-helpdesk`, `govoplan-identity`, `govoplan-inspections`,
|
`govoplan-identity`, `govoplan-identity-trust`, `govoplan-inspections`,
|
||||||
`govoplan-tickets`, `govoplan-learning`, `govoplan-permits`,
|
`govoplan-learning`, `govoplan-mandates`, `govoplan-parties`,
|
||||||
`govoplan-poll`, `govoplan-procurement`, `govoplan-records`,
|
`govoplan-permits`, `govoplan-poll`, `govoplan-procurement`,
|
||||||
`govoplan-resources`, `govoplan-rest`, `govoplan-risk-compliance`,
|
`govoplan-records`, `govoplan-resources`, `govoplan-rest`,
|
||||||
`govoplan-soap`, `govoplan-tenancy`, and `govoplan-transparency`.
|
`govoplan-services`, `govoplan-soap`, `govoplan-tickets`,
|
||||||
|
`govoplan-transparency`, `govoplan-wiki`, and `govoplan-workflow-engine`.
|
||||||
|
|
||||||
This is only negative route evidence. It does not classify the backend module's
|
Tenancy does provide composed administration surfaces despite having no direct
|
||||||
maturity or decide that it needs a WebUI. Connector-only, capability-only, or
|
route. This section is only negative package evidence; connector-only,
|
||||||
backend-only modules may remain intentionally headless.
|
capability-only, runtime-only, and backend-only modules may intentionally remain
|
||||||
|
headless. A new WebUI should be created only for a concrete user task, not to
|
||||||
|
make every module symmetrical.
|
||||||
|
|
||||||
## Rollout Matrix
|
## Rollout Matrix
|
||||||
|
|
||||||
@@ -208,17 +316,17 @@ backend-only modules may remain intentionally headless.
|
|||||||
| 0 | Product grammar and route inventory | Doctrine, ledger, layout rules, module contract, current route sources | One reconciled pattern language and evidence inventory | Meta [#11](https://git.add-ideas.de/GovOPlaN/govoplan/issues/11) | Docs links/diff checks; issue/wiki sync after integration | Initial slice in this document |
|
| 0 | Product grammar and route inventory | Doctrine, ledger, layout rules, module contract, current route sources | One reconciled pattern language and evidence inventory | Meta [#11](https://git.add-ideas.de/GovOPlaN/govoplan/issues/11) | Docs links/diff checks; issue/wiki sync after integration | Initial slice in this document |
|
||||||
| 1 | Campaign baseline integration | Recipient-editor WIP and tracker state have been reconciled with remote `main` | Integrated, testable baseline before migration claims | Campaign #67 and tracker cleanup | Backend and focused WebUI suites; issue evidence | Complete 2026-07-22 |
|
| 1 | Campaign baseline integration | Recipient-editor WIP and tracker state have been reconciled with remote `main` | Integrated, testable baseline before migration claims | Campaign #67 and tracker cleanup | Backend and focused WebUI suites; issue evidence | Complete 2026-07-22 |
|
||||||
| 2 | Campaign previews/details | Stable shared dialog with bounded scrolling and fixed responsive preview workspace | Stable header/body/footer, accessible long-content detail | Campaign #59 and #73 | Review-preview and overlay structure tests | Complete 2026-07-22 |
|
| 2 | Campaign previews/details | Stable shared dialog with bounded scrolling and fixed responsive preview workspace | Stable header/body/footer, accessible long-content detail | Campaign #59 and #73 | Review-preview and overlay structure tests | Complete 2026-07-22 |
|
||||||
| 3 | Campaign review/interventions | Five domain-owned stages with unresolved intervention language | Clear stages, outcomes, blockers, next actor/action, reviewed evidence | Campaign #63 | State matrix behavior/accessibility tests and agreed vocabulary | P1 needs product wording decision |
|
| 3 | Campaign review/interventions | Five domain-owned stages use central blocker and guided-review primitives; validation/build warnings name action, actor, and destination; hard blockers, individual review, and group review remain distinct; reviewed/remaining counts survive reload through build-bound review evidence | Clear stages, outcomes, blockers, next actor/action, reviewed evidence | Campaign #63 | `reviewProgress` state tests, shared-component structure contract, TypeScript build, configured-system help topic, and Campaign documentation tests | Complete 2026-08-03 (`d635f3a`; Core primitives and contextual help `b823a22`) |
|
||||||
| 4 | Campaign send/progress | A hard deployment ceiling bounds synchronous delivery; the selected synchronous, worker-queue, or database-queue mode is explicit and persisted; progress and recovery survive navigation; immediate-send response and audit evidence are allowlisted | Pre-send mode/consequence plus durable leave/return progress, retry and reconciliation without recipient/provider leakage | Campaign #62 and #79 | Boundary/concurrency/preflight, async selection, persisted mode, sanitized response/audit, partial/failure/retry and reload/return tests | Complete 2026-07-22 (`7e16603`, `60efd1c`, `62a6879`, `b0282eb`, `f095a3e`) |
|
| 4 | Campaign send/progress | A hard deployment ceiling bounds synchronous delivery; the selected synchronous, worker-queue, or database-queue mode is explicit and persisted; progress and recovery survive navigation; immediate-send response and audit evidence are allowlisted | Pre-send mode/consequence plus durable leave/return progress, retry and reconciliation without recipient/provider leakage | Campaign #62 and #79 | Boundary/concurrency/preflight, async selection, persisted mode, sanitized response/audit, partial/failure/retry and reload/return tests | Complete 2026-07-22 (`7e16603`, `60efd1c`, `62a6879`, `b0282eb`, `f095a3e`) |
|
||||||
| 5 | Campaign report filtering | Core DataGrid distinguishes client/full-result from server-owned queries; Campaign applies filter/sort/count before pagination and synchronizes count shortcuts with the grid query | One shared server-owned status/list/filter/count model | Campaign #65 and Core #263 | DataGrid contract/build tests plus exact shortcut/query/filter/count and large-result behavior | Complete 2026-07-22 (`e6062fe`, `cece71d`, `aa4ec66`, `4eb651c`) |
|
| 5 | Campaign report filtering | Core DataGrid distinguishes client/full-result from server-owned queries; Campaign applies filter/sort/count before pagination and synchronizes count shortcuts with the grid query | One shared server-owned status/list/filter/count model | Campaign #65 and Core #263 | DataGrid contract/build tests plus exact shortcut/query/filter/count and large-result behavior | Complete 2026-07-22 (`e6062fe`, `cece71d`, `aa4ec66`, `4eb651c`) |
|
||||||
| 6 | Campaign operator recovery | A durable campaign/version queue page exposes historical work, exact non-overlapping state counts, persisted mode, permission-safe controls, server-paged job evidence, bounded refresh and active-state recovery | Fixed-position actions, disabled explanations, leave/return state, version-scoped retry/queue/reconcile and explicit campaign-wide pause/resume/cancel | Campaign #78 | Queue model/structure, historical-version, permission, paging, recovery-control, stale-response and delta tests | Complete 2026-07-22 (`21f3014`, `99d44ee`, `735e874`) |
|
| 6 | Campaign operator recovery | A durable campaign/version queue page exposes historical work, exact non-overlapping state counts, persisted mode, permission-safe controls, server-paged job evidence, bounded refresh and active-state recovery | Fixed-position actions, disabled explanations, leave/return state, version-scoped retry/queue/reconcile and explicit campaign-wide pause/resume/cancel | Campaign #78 | Queue model/structure, historical-version, permission, paging, recovery-control, stale-response and delta tests | Complete 2026-07-22 (`21f3014`, `99d44ee`, `735e874`) |
|
||||||
| 7 | Campaign aggregate reports | A separate aggregate-reader projection and UI expose only policy-suppressed business totals with a stable status domain | Explicit denominator and exclusions, deployment floor plus tenant-strengthened small-cell threshold, complementary and overlapping-cell suppression, no detail/export/diagnostics | Campaign #80 | Aggregate query, cross-metric suppression, route/role/ACL, stable filter and UI structure tests | Complete 2026-07-22 (`06125cc`, `fc36aee`, `8ee87b7`, `ac3329c`, `1225802`) |
|
| 7 | Campaign aggregate reports | A separate aggregate-reader projection and UI expose only policy-suppressed business totals with a stable status domain | Explicit denominator and exclusions, deployment floor plus tenant-strengthened small-cell threshold, complementary and overlapping-cell suppression, no detail/export/diagnostics | Campaign #80 | Aggregate query, cross-metric suppression, route/role/ACL, stable filter and UI structure tests | Complete 2026-07-22 (`06125cc`, `fc36aee`, `8ee87b7`, `ac3329c`, `1225802`) |
|
||||||
| 8 | Campaign excluded outcomes | Excluded build rows become explicit skipped transport outcomes and remain protected from queue/cancel/retry ambiguity | One durable source-to-job-to-report meaning with guarded historical normalization | Campaign #66 | Builder/persistence, migration, query/count, queue-control and report-explanation tests | Complete 2026-07-22 (`7229fb8`) |
|
| 8 | Campaign excluded outcomes | Excluded build rows become explicit skipped transport outcomes and remain protected from queue/cancel/retry ambiguity | One durable source-to-job-to-report meaning with guarded historical normalization | Campaign #66 | Builder/persistence, migration, query/count, queue-control and report-explanation tests | Complete 2026-07-22 (`7229fb8`) |
|
||||||
| 9 | Guided first campaign | Existing wizard routes and ordinary workspace overlap | Task-oriented entry that hands off clearly to normal editing/review | Campaign #35 | First-run flow, resume/back, validation, optional modules, no implicit send | P1 after core pilot patterns stabilize |
|
| 9 | Guided first campaign | Existing wizard routes and ordinary workspace overlap | Task-oriented entry that hands off clearly to normal editing/review | Campaign #35 | First-run flow, resume/back, validation, optional modules, no implicit send | P1 after core pilot patterns stabilize |
|
||||||
| 10 | Prove/extract generic primitives | Core already exports many primitives; Campaign composition still unreviewed | Extract only contracts with a second consumer or clear platform ownership | Core #225 plus bounded follow-ups | Core behavior/accessibility tests and module-permutation tests | After Campaign proof |
|
| 10 | Prove/extract generic primitives | Shared consequence, focus, help, blocker, unsaved-change, confirmation, connection-tree and effective-policy contracts now have Core and multiple module consumers | Keep Core behavior-only and leave domain composition in owning modules | Core #225 plus bounded follow-ups | Core behavior/accessibility tests and module-permutation tests | Complete 2026-08-03 (`fa32cca`; Files `d8ae506`; Mail `7844d9c`) |
|
||||||
| 11 | Configured-system pattern help | Docs route and classification exist | Role/config-aware pattern and route/field/blocker help | Docs #15 | Topic grouping, audience filtering, stable links/anchors | P1 after initial pattern IDs stabilize |
|
| 11 | Configured-system pattern help | Role/config-aware workflow, reference, pattern, and system topics are projected by Docs; shared route, field, blocker, and action links resolve to configured Docs or the hosted fallback | Stable configured-system guidance without feature-to-Docs imports | Docs #15 | Docs suite, shared component tests, Campaign review tests, 46 module permutations, full-product bundle budget | Complete 2026-08-03 (Docs `abe2f78`; Core `b823a22`; Campaign `d635f3a`) |
|
||||||
| 12 | Admin/configuration family | Phase inventory and connector primitives exist in the ledger | Apply the pattern to files, mail, policy, retention, packages, modules, API keys, settings | Core #225 and module children | Per-surface state/accessibility/consequence evidence | Parallel where independent of Campaign shared decisions |
|
| 12 | Admin/configuration family | Core host/settings/credential/retention contracts, shared primitives, module lifecycle, Files, Mail, Policy, Access, Admin, Tenancy, Views, and Organizations are integrated and verified | Continue the same consequence/provenance grammar only through bounded module-owned migrations | Core #225 and module children | Per-surface state/accessibility/consequence evidence | Core #225 complete `fa32cca`; Access `1409dbf`; Files `d8ae506`; Mail `7844d9c`; Policy `f964ed7`; Admin `d428f33`; Tenancy `e76fe16`; Views `c125f33`; Organizations `97acfcb` |
|
||||||
| 13 | Remaining direct routes | Routes are contributed; most are unreviewed | Per-module bounded audit and migration plan | New module issues derived from this inventory | Applicable definition-of-done gates | P2 after Campaign, not a bulk rewrite |
|
| 13 | Remaining module surfaces | 33 bounded module-owned issues cover every WebUI contributor not already tracked by Campaign #74 or completed Docs #15 | Per-module audit and migration, ordered by user task and consequence rather than a bulk rewrite | Issues linked in the direct-route and composed-surface sections | Module-focused tests, manifest shapes, contextual Docs, and applicable definition-of-done gates | Scheduling `c17cbda`, Audit `6d3fcc1`, Access `1409dbf`, Files `d8ae506`, Mail `7844d9c`, Policy `f964ed7`, Admin `d428f33`, Tenancy `e76fe16`, Views `c125f33`, Organizations `97acfcb`, Postbox `a97eb3b`, IDM `d864317`, Committee `e64af30`, Approvals `24e9559`, Forms Runtime `07dd35b`, Forms `e505536`, Voting `2625990`, Distribution Lists `6cdd804`, Templates `72fafa2`, Addresses `f9a7185`, Datasources `6406ce7`, Dataflow `109ddcd`, Dashboard `da3947f`, Cases `43b4cc8`, Calendar `d7fd944`, Ops `2b32643`, Notifications `ad6a31f`, and Risk Compliance `24d80a6` complete |
|
||||||
| 14 | Manifest/runtime alignment | Several executable routes are absent from manifest metadata | Declared alignment or explicit validated exception | Core contract issue to create | Automated manifest/module route check and configured Docs verification | Discovery follow-up |
|
| 14 | Manifest/runtime alignment | Several executable routes are absent from manifest metadata | Declared alignment or explicit validated exception | Core contract issue to create | Automated manifest/module route check and configured Docs verification | Discovery follow-up |
|
||||||
|
|
||||||
Workflow remains outside this rollout matrix because it has its own runtime and
|
Workflow remains outside this rollout matrix because it has its own runtime and
|
||||||
|
|||||||
@@ -44,6 +44,10 @@ least one check. The ledger verifies its hash chain before evidence is trusted.
|
|||||||
This is a platform contract, not an assertion that every existing module
|
This is a platform contract, not an assertion that every existing module
|
||||||
operation has adopted it. Module operations with external or multi-resource
|
operation has adopted it. Module operations with external or multi-resource
|
||||||
effects must be migrated to the ledger before claiming these guarantees.
|
effects must be migrated to the ledger before claiming these guarantees.
|
||||||
|
The owning-module inventory and adoption state are maintained in
|
||||||
|
[Recovery Ledger Adoption](RECOVERY_LEDGER_ADOPTION.md); CI validates the
|
||||||
|
machine-readable inventory so newly identified boundaries cannot disappear from
|
||||||
|
the backlog silently.
|
||||||
|
|
||||||
## Deployment Journal
|
## Deployment Journal
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
# Recovery Ledger Adoption
|
||||||
|
|
||||||
|
The Core recovery ledger is a platform primitive, not automatic protection for
|
||||||
|
module-owned effects. The canonical, machine-checked inventory is
|
||||||
|
[`recovery-operation-inventory.json`](recovery-operation-inventory.json).
|
||||||
|
|
||||||
|
## Classification Rules
|
||||||
|
|
||||||
|
- Use `atomic` only when every mutation commits in one database transaction and
|
||||||
|
no external effect occurs.
|
||||||
|
- Use `compensation` when every completed effect has a bounded, verifiable
|
||||||
|
inverse action. A best-effort delete is not proof of compensation.
|
||||||
|
- Use `snapshot_restore` only with fresh, signed backup evidence that covers all
|
||||||
|
affected state services at one recovery point.
|
||||||
|
- Use `forward_recovery` for provider acceptance, queue publication, cursor
|
||||||
|
advancement, and other effects that may be resumable but cannot safely be
|
||||||
|
undone.
|
||||||
|
- Use `irreversible` for approved purge or destruction where no automated
|
||||||
|
recovery is claimed.
|
||||||
|
|
||||||
|
One feature may cross more than one boundary. Module installation is
|
||||||
|
compensatable before schema migration, forward-only after migration starts, and
|
||||||
|
snapshot-restorable for an approved destructive retirement. Mail submission is
|
||||||
|
forward recovery because losing the response after provider acceptance must not
|
||||||
|
cause an automatic resend.
|
||||||
|
|
||||||
|
## Adoption Order
|
||||||
|
|
||||||
|
1. Campaign build is the reference implementation for a database plus object
|
||||||
|
storage operation. Its operation reserves a build-specific object prefix,
|
||||||
|
persists request and precondition evidence before writes, records the final
|
||||||
|
object manifest, and verifies database/object state before success.
|
||||||
|
2. Campaign delivery and Mail provider effects adopt outcome-unknown semantics
|
||||||
|
without weakening their existing provider-specific idempotency records.
|
||||||
|
3. Files applies the same contract to uploads, purge, integrity reconciliation,
|
||||||
|
and writable connector synchronization.
|
||||||
|
4. Connectors, Dataflow, and Workflow Engine consume the contract at their
|
||||||
|
registry/capability boundaries so optional providers remain optional.
|
||||||
|
5. Core module lifecycle uses the ledger in addition to, not instead of, signed
|
||||||
|
deployment and backup evidence.
|
||||||
|
|
||||||
|
Every fenced operation uses a process incarnation and distributed lease. A
|
||||||
|
stale process cannot append a checkpoint or report success. An expired operation
|
||||||
|
is claimed for recovery through an explicit takeover that preserves the prior
|
||||||
|
fence in the checkpoint chain; it is never resumed as a normal retry.
|
||||||
|
|
||||||
|
Connectors read-only sanctions and feed acquisitions are adopted: source
|
||||||
|
revision/cursor and dry-run evidence are recorded before provider I/O, while
|
||||||
|
the immutable snapshot and terminal checkpoint commit atomically. The generic
|
||||||
|
external-mutation contract is conformance-tested but remains `planned` until a
|
||||||
|
production connector actually publishes, updates, or deletes provider state.
|
||||||
|
|
||||||
|
Dataflow runs are adopted. Database-only execution uses one atomic terminal
|
||||||
|
commit for the run projection and recovery checkpoint. Output publication uses
|
||||||
|
forward recovery: source and output digests are checkpointed before dispatch,
|
||||||
|
a conclusive provider result commits with the run projection, and an expired
|
||||||
|
or failed attempt after dispatch becomes `outcome_unknown`. A stale attempt may
|
||||||
|
be retried only when its durable boundary proves dispatch had not started.
|
||||||
|
|
||||||
|
Workflow Engine is adopted at both declared boundaries. Instance workers,
|
||||||
|
trigger deliveries, and timer resumptions use process-bound distributed fences.
|
||||||
|
Every module-action invocation records the pinned definition, input, preview,
|
||||||
|
authority, provider-idempotency, and action-contract hashes before dispatch.
|
||||||
|
Conclusive results commit with the Workflow projection. A lost acknowledgement,
|
||||||
|
invalid result, or unannounced non-atomic effect becomes `outcome_unknown` and
|
||||||
|
cannot be retried until evidence confirms either that the effect occurred or is
|
||||||
|
absent. Linked Dataflow uncertainty blocks the Workflow without duplicating
|
||||||
|
Dataflow's recovery authority.
|
||||||
|
|
||||||
|
Core module lifecycle is adopted at four boundaries. Installer recovery is
|
||||||
|
prepared before snapshots so a full database restore preserves the attempted
|
||||||
|
operation. Pre-migration package changes use compensation, migrated changes use
|
||||||
|
forward recovery, destructive retirement requires a hashed and restore-checked
|
||||||
|
snapshot, and live graph changes restore the prior registry when no migration
|
||||||
|
ran. A deployment-wide database fence serializes these effects; any unresolved
|
||||||
|
predecessor blocks a differently keyed retry until explicit reconciliation.
|
||||||
|
Supervised installs become successful only after restart and health evidence is
|
||||||
|
recorded.
|
||||||
|
|
||||||
|
## Operator Contract
|
||||||
|
|
||||||
|
Ops lists non-terminal and manual-intervention operations. Operators must verify
|
||||||
|
the checkpoint chain before trusting evidence, distinguish `outcome_unknown`
|
||||||
|
from rejection, and use the owning module's documented reconciliation action.
|
||||||
|
No evidence payload may contain credentials or resolved secrets.
|
||||||
|
|
||||||
|
The parent adoption issue remains open until all inventory rows are adopted and
|
||||||
|
the module matrix proves crash, retry, stale-fence, tamper, and optional-module
|
||||||
|
behavior for each consequential path.
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"parent_issue": "https://git.add-ideas.de/GovOPlaN/govoplan/issues/36",
|
||||||
|
"operations": [
|
||||||
|
{
|
||||||
|
"id": "campaign.build.publish-artifacts",
|
||||||
|
"repository": "govoplan-campaign",
|
||||||
|
"resources": ["postgresql", "object-storage", "templates-capability", "files-capability"],
|
||||||
|
"mode": "compensation",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "reference-implementation",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "campaign.delivery.external-channels",
|
||||||
|
"repository": "govoplan-campaign",
|
||||||
|
"resources": ["postgresql", "queue", "smtp", "imap", "postbox", "print-provider"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "campaign.retention.generated-artifacts",
|
||||||
|
"repository": "govoplan-campaign",
|
||||||
|
"resources": ["postgresql", "object-storage"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-campaign/issues/92"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "files.upload.finalize",
|
||||||
|
"repository": "govoplan-files",
|
||||||
|
"resources": ["postgresql", "object-storage", "filesystem-staging"],
|
||||||
|
"mode": "compensation",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "files.retention.purge",
|
||||||
|
"repository": "govoplan-files",
|
||||||
|
"resources": ["postgresql", "object-storage", "encryption-key-custody"],
|
||||||
|
"mode": "irreversible",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "planned",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "files.integrity.reconcile",
|
||||||
|
"repository": "govoplan-files",
|
||||||
|
"resources": ["postgresql", "object-storage"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "files.connector.write-sync",
|
||||||
|
"repository": "govoplan-files",
|
||||||
|
"resources": ["postgresql", "object-storage", "external-connector"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "planned",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/41"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "mail.outbox.smtp-submit",
|
||||||
|
"repository": "govoplan-mail",
|
||||||
|
"resources": ["postgresql", "queue", "smtp"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "mail.sent.imap-append",
|
||||||
|
"repository": "govoplan-mail",
|
||||||
|
"resources": ["postgresql", "imap"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "mail.mailbox.imap-mutate",
|
||||||
|
"repository": "govoplan-mail",
|
||||||
|
"resources": ["postgresql", "imap"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "planned",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "mail.mailbox.sync-cursor",
|
||||||
|
"repository": "govoplan-mail",
|
||||||
|
"resources": ["postgresql", "imap"],
|
||||||
|
"mode": "atomic",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-mail/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "connectors.sync.read-snapshot",
|
||||||
|
"repository": "govoplan-connectors",
|
||||||
|
"resources": ["postgresql", "external-provider"],
|
||||||
|
"mode": "atomic",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "connectors.sync.external-mutation",
|
||||||
|
"repository": "govoplan-connectors",
|
||||||
|
"resources": ["postgresql", "queue", "external-provider"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "planned",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-connectors/issues/15"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "dataflow.run.database-only",
|
||||||
|
"repository": "govoplan-dataflow",
|
||||||
|
"resources": ["postgresql"],
|
||||||
|
"mode": "atomic",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "dataflow.run.publish-output",
|
||||||
|
"repository": "govoplan-dataflow",
|
||||||
|
"resources": ["postgresql", "queue", "object-storage", "external-sink"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-dataflow/issues/19"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "workflow-engine.instance.state-transition",
|
||||||
|
"repository": "govoplan-workflow-engine",
|
||||||
|
"resources": ["postgresql"],
|
||||||
|
"mode": "atomic",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "workflow-engine.activity.external-effect",
|
||||||
|
"repository": "govoplan-workflow-engine",
|
||||||
|
"resources": ["postgresql", "queue", "module-capability", "external-provider"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-workflow-engine/issues/1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "core.module-lifecycle.pre-migration",
|
||||||
|
"repository": "govoplan-core",
|
||||||
|
"resources": ["postgresql", "package-environment", "webui-bundle", "filesystem"],
|
||||||
|
"mode": "compensation",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "core.module-lifecycle.post-migration",
|
||||||
|
"repository": "govoplan-core",
|
||||||
|
"resources": ["postgresql", "package-environment", "webui-bundle", "runtime-nodes"],
|
||||||
|
"mode": "forward_recovery",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "core.module-retirement.destroy-data",
|
||||||
|
"repository": "govoplan-core",
|
||||||
|
"resources": ["postgresql", "object-storage", "package-environment"],
|
||||||
|
"mode": "snapshot_restore",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "core.module-runtime.apply-graph",
|
||||||
|
"repository": "govoplan-core",
|
||||||
|
"resources": ["postgresql", "runtime-nodes", "module-registry"],
|
||||||
|
"mode": "compensation",
|
||||||
|
"fenced": true,
|
||||||
|
"adoption": "adopted",
|
||||||
|
"issue": "https://git.add-ideas.de/GovOPlaN/govoplan-core/issues/281"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -551,6 +551,9 @@ class DeploymentInstallerTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertIn("caddy-data:/data", ingress["volumes"])
|
self.assertIn("caddy-data:/data", ingress["volumes"])
|
||||||
self.assertIn("caddy-config:/config", ingress["volumes"])
|
self.assertIn("caddy-config:/config", ingress["volumes"])
|
||||||
|
self.assertEqual(["ALL"], ingress["cap_drop"])
|
||||||
|
self.assertEqual(["NET_BIND_SERVICE"], ingress["cap_add"])
|
||||||
|
self.assertEqual(["no-new-privileges:true"], ingress["security_opt"])
|
||||||
self.assertIn("reverse_proxy load-balancer:8080", render_caddy_config(spec))
|
self.assertIn("reverse_proxy load-balancer:8080", render_caddy_config(spec))
|
||||||
self.assertNotIn("operator@example.test", json.dumps(compose))
|
self.assertNotIn("operator@example.test", json.dumps(compose))
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _load_module():
|
||||||
|
path = ROOT / "tools/checks/managed-ingress-drill.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("managed_ingress_drill", path)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
INGRESS = _load_module()
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedIngressDrillTests(unittest.TestCase):
|
||||||
|
def test_config_is_streamed_into_a_daemon_visible_volume(self) -> None:
|
||||||
|
completed = subprocess.CompletedProcess([], 0, "", "")
|
||||||
|
with patch.object(INGRESS, "_run", return_value=completed) as run:
|
||||||
|
INGRESS._write_volume_file(
|
||||||
|
image="registry.example/caddy@sha256:" + "1" * 64,
|
||||||
|
volume="config-volume",
|
||||||
|
filename="Caddyfile",
|
||||||
|
content=":8080 { respond /health 200 }\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
argv = run.call_args.args[0]
|
||||||
|
self.assertIn("type=volume,src=config-volume,dst=/govoplan-config", argv)
|
||||||
|
self.assertIn("0:0", argv)
|
||||||
|
self.assertNotIn("type=bind", " ".join(argv))
|
||||||
|
self.assertEqual(
|
||||||
|
":8080 { respond /health 200 }\n",
|
||||||
|
run.call_args.kwargs["input_text"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_config_filename_cannot_escape_the_volume(self) -> None:
|
||||||
|
with self.assertRaisesRegex(ValueError, "invalid config filename"):
|
||||||
|
INGRESS._write_volume_file(
|
||||||
|
image="registry.example/caddy@sha256:" + "1" * 64,
|
||||||
|
volume="config-volume",
|
||||||
|
filename="../Caddyfile",
|
||||||
|
content="",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_drill_has_no_runner_local_bind_mounts(self) -> None:
|
||||||
|
source = (ROOT / "tools/checks/managed-ingress-drill.py").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertNotIn("type=bind", source)
|
||||||
|
self.assertIn('"--network-alias",\n "load-balancer"', source)
|
||||||
|
self.assertNotIn('"127.0.0.1::8080"', source)
|
||||||
|
self.assertIn("requested_http_port", source)
|
||||||
|
self.assertIn("requested_https_port", source)
|
||||||
|
self.assertIn('"--cap-add",\n "NET_BIND_SERVICE"', source)
|
||||||
|
|
||||||
|
def test_published_port_reads_the_docker_mapping(self) -> None:
|
||||||
|
completed = subprocess.CompletedProcess(
|
||||||
|
[],
|
||||||
|
0,
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"8443/tcp": [
|
||||||
|
{"HostIp": "127.0.0.1", "HostPort": "49152"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
with patch.object(INGRESS, "_run", return_value=completed) as run:
|
||||||
|
port = INGRESS._published_port("ingress", 8443)
|
||||||
|
|
||||||
|
self.assertEqual(49152, port)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"inspect",
|
||||||
|
"--format",
|
||||||
|
"{{json .HostConfig.PortBindings}}",
|
||||||
|
"ingress",
|
||||||
|
],
|
||||||
|
run.call_args.args[0],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_published_port_rejects_non_loopback_binding(self) -> None:
|
||||||
|
completed = subprocess.CompletedProcess(
|
||||||
|
[],
|
||||||
|
0,
|
||||||
|
'{"8443/tcp":[{"HostIp":"0.0.0.0","HostPort":"49152"}]}',
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
with patch.object(INGRESS, "_run", return_value=completed):
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "loopback binding"):
|
||||||
|
INGRESS._published_port("ingress", 8443)
|
||||||
|
|
||||||
|
def test_probe_runs_as_a_network_sibling_from_a_digest_image(self) -> None:
|
||||||
|
completed = subprocess.CompletedProcess([], 0, "", "")
|
||||||
|
image = "registry.example/runtime-api@sha256:" + "1" * 64
|
||||||
|
with patch.object(INGRESS, "_run", return_value=completed) as run:
|
||||||
|
INGRESS._probe_ingress(
|
||||||
|
image=image,
|
||||||
|
network="deployment-network",
|
||||||
|
container="ingress",
|
||||||
|
)
|
||||||
|
|
||||||
|
argv = run.call_args.args[0]
|
||||||
|
self.assertEqual("docker", argv[0])
|
||||||
|
self.assertIn("deployment-network", argv)
|
||||||
|
self.assertIn(image, argv)
|
||||||
|
self.assertIn('(\"ingress\", port)', argv[-1])
|
||||||
|
self.assertIn("server_hostname=\"localhost\"", argv[-1])
|
||||||
|
self.assertNotIn("localhost:49152", argv[-1])
|
||||||
|
|
||||||
|
def test_probe_diagnostics_include_container_stderr(self) -> None:
|
||||||
|
probe_failure = subprocess.CalledProcessError(1, ["docker", "run"])
|
||||||
|
state = subprocess.CompletedProcess([], 0, '{"Running":false}', "")
|
||||||
|
logs = subprocess.CompletedProcess([], 0, "", "caddy startup failed")
|
||||||
|
with patch.object(
|
||||||
|
INGRESS,
|
||||||
|
"_run",
|
||||||
|
side_effect=[probe_failure, state, logs],
|
||||||
|
), patch.object(INGRESS.sys, "stderr") as stderr:
|
||||||
|
with self.assertRaises(subprocess.CalledProcessError):
|
||||||
|
INGRESS._probe_ingress(
|
||||||
|
image="registry.example/runtime-api@sha256:" + "1" * 64,
|
||||||
|
network="deployment-network",
|
||||||
|
container="ingress",
|
||||||
|
)
|
||||||
|
|
||||||
|
rendered = "".join(call.args[0] for call in stderr.write.call_args_list)
|
||||||
|
self.assertIn('"Running":false', rendered)
|
||||||
|
self.assertIn("caddy startup failed", rendered)
|
||||||
|
|
||||||
|
def test_standalone_workflow_is_dispatch_only_and_digest_bounded(self) -> None:
|
||||||
|
workflow = (
|
||||||
|
ROOT / ".gitea/workflows/runtime-ingress-drill.yml"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
self.assertIn("workflow_dispatch:", workflow)
|
||||||
|
self.assertNotIn("\n push:", workflow)
|
||||||
|
self.assertIn("--probe-image \"$PROBE_IMAGE\"", workflow)
|
||||||
|
self.assertIn("GOVOPLAN_REGISTRY_TOKEN", workflow)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -113,6 +113,60 @@ class PythonEnvironmentSyncTests(unittest.TestCase):
|
|||||||
self.assertEqual(plan.mode, "Selective Python environment repair")
|
self.assertEqual(plan.mode, "Selective Python environment repair")
|
||||||
self.assertEqual(plan.commands[0][-2:], ("-e", str(project_root)))
|
self.assertEqual(plan.commands[0][-2:], ("-e", str(project_root)))
|
||||||
|
|
||||||
|
def test_metadata_sync_also_repairs_unrelated_missing_distribution(self) -> None:
|
||||||
|
sync = load_sync_module()
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-python-sync-") as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
requirements = root / "requirements-dev.txt"
|
||||||
|
requirements.write_text("-e ./govoplan-changed\n-e ./govoplan-missing\n", encoding="utf-8")
|
||||||
|
for project in ("govoplan-changed", "govoplan-missing"):
|
||||||
|
project_root = root / project
|
||||||
|
project_root.mkdir()
|
||||||
|
(project_root / "pyproject.toml").write_text(
|
||||||
|
f'[project]\nname = "{project}"\nversion = "0.1.10"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
entries = sync.local_requirement_entries(requirements)
|
||||||
|
fingerprint = sync.build_fingerprint(
|
||||||
|
requirements=requirements,
|
||||||
|
python="/test/venv/bin/python",
|
||||||
|
local_requirements=entries,
|
||||||
|
)
|
||||||
|
requirements_digest = hashlib.sha256(requirements.read_bytes()).hexdigest()
|
||||||
|
previous = {
|
||||||
|
"version": sync.STAMP_VERSION,
|
||||||
|
"python": "/test/venv/bin/python",
|
||||||
|
"inputs": [
|
||||||
|
{"path": str(requirements), "sha256": requirements_digest},
|
||||||
|
{"path": entries[0].pyproject, "sha256": "stale"},
|
||||||
|
{
|
||||||
|
"path": entries[1].pyproject,
|
||||||
|
"sha256": hashlib.sha256(Path(entries[1].pyproject).read_bytes()).hexdigest(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"requirements_entries": [
|
||||||
|
entry.as_dict() for entry in sync.parse_requirement_entries(requirements)
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
plan = sync.build_install_plan(
|
||||||
|
previous=previous,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
requirements=requirements,
|
||||||
|
python="/test/venv/bin/python",
|
||||||
|
local_requirements=entries,
|
||||||
|
repair_requirements=(entries[1],),
|
||||||
|
force=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(plan.mode, "Selective Python environment sync")
|
||||||
|
self.assertEqual(len(plan.commands), 1)
|
||||||
|
command = plan.commands[0]
|
||||||
|
self.assertEqual(command.count("-e"), 2)
|
||||||
|
self.assertIn(str(root / "govoplan-changed"), command)
|
||||||
|
self.assertIn(str(root / "govoplan-missing"), command)
|
||||||
|
|
||||||
def test_declared_module_entry_points_are_part_of_environment_validation(self) -> None:
|
def test_declared_module_entry_points_are_part_of_environment_validation(self) -> None:
|
||||||
sync = load_sync_module()
|
sync = load_sync_module()
|
||||||
with tempfile.TemporaryDirectory(prefix="govoplan-python-sync-") as directory:
|
with tempfile.TemporaryDirectory(prefix="govoplan-python-sync-") as directory:
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
INVENTORY = ROOT / "docs" / "recovery-operation-inventory.json"
|
||||||
|
MODES = {
|
||||||
|
"atomic",
|
||||||
|
"compensation",
|
||||||
|
"snapshot_restore",
|
||||||
|
"forward_recovery",
|
||||||
|
"irreversible",
|
||||||
|
}
|
||||||
|
ADOPTION_STATES = {"planned", "reference-implementation", "adopted"}
|
||||||
|
REQUIRED_PREFIXES = {
|
||||||
|
"campaign.",
|
||||||
|
"files.",
|
||||||
|
"mail.",
|
||||||
|
"connectors.",
|
||||||
|
"dataflow.",
|
||||||
|
"workflow-engine.",
|
||||||
|
"core.module-lifecycle.",
|
||||||
|
"core.module-runtime.",
|
||||||
|
}
|
||||||
|
ATOMIC_EXTERNAL_READS = {
|
||||||
|
"connectors.sync.read-snapshot",
|
||||||
|
"mail.mailbox.sync-cursor",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_operation_inventory_is_complete_and_actionable() -> None:
|
||||||
|
payload = json.loads(INVENTORY.read_text(encoding="utf-8"))
|
||||||
|
assert payload["schema_version"] == 1
|
||||||
|
operations = payload["operations"]
|
||||||
|
ids = [item["id"] for item in operations]
|
||||||
|
assert len(ids) == len(set(ids))
|
||||||
|
assert all(any(item.startswith(prefix) for item in ids) for prefix in REQUIRED_PREFIXES)
|
||||||
|
|
||||||
|
for item in operations:
|
||||||
|
assert item["mode"] in MODES
|
||||||
|
assert item["adoption"] in ADOPTION_STATES
|
||||||
|
assert item["repository"].startswith("govoplan-")
|
||||||
|
assert item["resources"]
|
||||||
|
assert item["fenced"] is True
|
||||||
|
issue = urlparse(item["issue"])
|
||||||
|
assert issue.scheme == "https"
|
||||||
|
assert issue.netloc == "git.add-ideas.de"
|
||||||
|
assert issue.path.startswith(f"/GovOPlaN/{item['repository']}/issues/")
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_atomic_operations_do_not_claim_plain_database_rollback() -> None:
|
||||||
|
operations = json.loads(INVENTORY.read_text(encoding="utf-8"))["operations"]
|
||||||
|
for item in operations:
|
||||||
|
if item["mode"] == "atomic":
|
||||||
|
assert (
|
||||||
|
item["resources"] == ["postgresql"]
|
||||||
|
or item["id"] in ATOMIC_EXTERNAL_READS
|
||||||
|
)
|
||||||
@@ -3,10 +3,14 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
@@ -26,9 +30,211 @@ FINALIZE = _load(
|
|||||||
"finalize_runtime_distribution",
|
"finalize_runtime_distribution",
|
||||||
ROOT / "tools/release/finalize-runtime-distribution.py",
|
ROOT / "tools/release/finalize-runtime-distribution.py",
|
||||||
)
|
)
|
||||||
|
DEPLOYER_BUILD = _load(
|
||||||
|
"build_deployer_zipapp",
|
||||||
|
ROOT / "tools/deployment/build-deployer-zipapp.py",
|
||||||
|
)
|
||||||
|
PUBLISH = _load(
|
||||||
|
"publish_runtime_release",
|
||||||
|
ROOT / "tools/release/publish-runtime-release.py",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RuntimeDistributionBuildTests(unittest.TestCase):
|
class RuntimeDistributionBuildTests(unittest.TestCase):
|
||||||
|
def test_deployment_zipapp_is_reproducible_across_source_mtimes(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-reproducible-zipapp-") as value:
|
||||||
|
root = Path(value)
|
||||||
|
source = root / "source"
|
||||||
|
shutil.copytree(ROOT / "tools/deployment", source)
|
||||||
|
first = root / "first.pyz"
|
||||||
|
second = root / "second.pyz"
|
||||||
|
original_root = DEPLOYER_BUILD.ROOT
|
||||||
|
try:
|
||||||
|
DEPLOYER_BUILD.ROOT = source
|
||||||
|
self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(first)]))
|
||||||
|
for path in source.rglob("*.py"):
|
||||||
|
os.utime(path, (2_000_000_000, 2_000_000_000))
|
||||||
|
self.assertEqual(0, DEPLOYER_BUILD.main(["--output", str(second)]))
|
||||||
|
finally:
|
||||||
|
DEPLOYER_BUILD.ROOT = original_root
|
||||||
|
|
||||||
|
self.assertEqual(first.read_bytes(), second.read_bytes())
|
||||||
|
|
||||||
|
def test_workflow_signs_with_the_release_environment(self) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn(
|
||||||
|
".runtime-build/bin/python tools/release/generate-runtime-distribution.py",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
self.assertNotIn(
|
||||||
|
"\n python tools/release/generate-runtime-distribution.py",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_workflow_rejects_missing_or_mutable_image_inputs_before_build(self) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
validation = workflow.index("- name: Validate immutable release inputs")
|
||||||
|
bootstrap = workflow.index("- name: Bootstrap release sources")
|
||||||
|
self.assertLess(validation, bootstrap)
|
||||||
|
self.assertIn('image_pattern = re.compile(r"^[^@\\s]+@sha256:', workflow)
|
||||||
|
for input_name in (
|
||||||
|
"python_image",
|
||||||
|
"nginx_image",
|
||||||
|
"postgres_image",
|
||||||
|
"redis_image",
|
||||||
|
"load_balancer_image",
|
||||||
|
"managed_ingress_image",
|
||||||
|
"garage_image",
|
||||||
|
"test_mail_image",
|
||||||
|
"binfmt_image",
|
||||||
|
):
|
||||||
|
self.assertIn(f"inputs.{input_name}", workflow)
|
||||||
|
|
||||||
|
def test_api_runtime_points_core_at_packaged_migration_scripts(self) -> None:
|
||||||
|
dockerfile = (ROOT / "tools/release/runtime/Dockerfile.api").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn(
|
||||||
|
"GOVOPLAN_CORE_SOURCE_ROOT=/opt/govoplan/runtime/govoplan_core_runtime",
|
||||||
|
dockerfile,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_web_runtime_uses_only_writable_tmpfs_for_nginx_temp_files(self) -> None:
|
||||||
|
nginx = (ROOT / "tools/release/runtime/nginx.conf").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
for temporary_path in (
|
||||||
|
"client_body_temp_path /tmp/client_temp;",
|
||||||
|
"fastcgi_temp_path /tmp/fastcgi_temp;",
|
||||||
|
"proxy_temp_path /tmp/proxy_temp;",
|
||||||
|
"scgi_temp_path /tmp/scgi_temp;",
|
||||||
|
"uwsgi_temp_path /tmp/uwsgi_temp;",
|
||||||
|
):
|
||||||
|
self.assertIn(temporary_path, nginx)
|
||||||
|
|
||||||
|
def test_workflow_verifies_portable_bootstrap_artifacts_before_execution(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn(
|
||||||
|
"(cd runtime-output && sha256sum govoplan-deploy.pyz > "
|
||||||
|
"govoplan-deploy.pyz.sha256)",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
self.assertIn("openssl pkeyutl -verify -pubin", workflow)
|
||||||
|
self.assertIn("govoplan-deploy.tampered.pyz", workflow)
|
||||||
|
self.assertLess(
|
||||||
|
workflow.index("openssl pkeyutl -verify -pubin"),
|
||||||
|
workflow.index("python runtime-output/govoplan-deploy.pyz init"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_workflow_retains_both_platform_runtime_smoke_receipts(self) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn('for ARCH in amd64 arm64; do', workflow)
|
||||||
|
self.assertIn("tools/checks/runtime-image-smoke.py", workflow)
|
||||||
|
self.assertIn("Resolve managed dependency platform images", workflow)
|
||||||
|
self.assertIn("--postgres-metadata", workflow)
|
||||||
|
self.assertIn("--redis-metadata", workflow)
|
||||||
|
self.assertIn("Register arm64 execution for runtime smoke", workflow)
|
||||||
|
self.assertIn(
|
||||||
|
'docker run --privileged --rm "$BINFMT_IMAGE" --install arm64',
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
self.assertIn("runtime-smoke-amd64.json", workflow)
|
||||||
|
self.assertIn("runtime-smoke-arm64.json", workflow)
|
||||||
|
self.assertIn(
|
||||||
|
"jq -r '.platforms[\"linux/amd64\"]' runtime-output/api-metadata.json",
|
||||||
|
workflow,
|
||||||
|
)
|
||||||
|
self.assertNotIn(".platforms[\\\"linux/amd64\\\"]", workflow)
|
||||||
|
|
||||||
|
def test_workflow_binds_the_release_tag_to_the_workflow_commit(self) -> None:
|
||||||
|
workflow = (ROOT / ".gitea/workflows/runtime-distribution.yml").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
publisher = (ROOT / "tools/release/publish-runtime-release.py").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn("SOURCE_COMMIT: ${{ gitea.sha }}", workflow)
|
||||||
|
self.assertIn('--target-commit "$SOURCE_COMMIT"', workflow)
|
||||||
|
self.assertIn('"target_commitish": target_commit', publisher)
|
||||||
|
self.assertIn("self._resolve_commit(tag) != target_commit", publisher)
|
||||||
|
|
||||||
|
def test_runtime_publisher_rejects_a_tag_on_another_commit(self) -> None:
|
||||||
|
publisher = PUBLISH.GiteaReleasePublisher(
|
||||||
|
base_url="https://git.example.test",
|
||||||
|
owner="GovOPlaN",
|
||||||
|
repo="govoplan",
|
||||||
|
token="secret",
|
||||||
|
)
|
||||||
|
target = "1" * 40
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
publisher,
|
||||||
|
"_resolve_commit",
|
||||||
|
side_effect=(target, "2" * 40),
|
||||||
|
),
|
||||||
|
self.assertRaisesRegex(PUBLISH.PublishError, "another commit"),
|
||||||
|
):
|
||||||
|
publisher.release(
|
||||||
|
tag="v1.2.3",
|
||||||
|
target_commit=target,
|
||||||
|
title="Release",
|
||||||
|
body="Body",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_runtime_publisher_creates_the_tag_at_the_exact_commit(self) -> None:
|
||||||
|
publisher = PUBLISH.GiteaReleasePublisher(
|
||||||
|
base_url="https://git.example.test",
|
||||||
|
owner="GovOPlaN",
|
||||||
|
repo="govoplan",
|
||||||
|
token="secret",
|
||||||
|
)
|
||||||
|
target = "1" * 40
|
||||||
|
requests: list[tuple[str, dict[str, object] | None]] = []
|
||||||
|
|
||||||
|
def request(method: str, _url: str, **kwargs):
|
||||||
|
payload = kwargs.get("payload")
|
||||||
|
requests.append((method, payload))
|
||||||
|
if method == "GET":
|
||||||
|
raise HTTPError(_url, 404, "not found", {}, None)
|
||||||
|
return {"id": 1}
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
publisher,
|
||||||
|
"_resolve_commit",
|
||||||
|
side_effect=(target, None, target),
|
||||||
|
),
|
||||||
|
patch.object(publisher, "_json", side_effect=request),
|
||||||
|
):
|
||||||
|
release = publisher.release(
|
||||||
|
tag="v1.2.3",
|
||||||
|
target_commit=target,
|
||||||
|
title="Release",
|
||||||
|
body="Body",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual({"id": 1}, release)
|
||||||
|
self.assertEqual("POST", requests[-1][0])
|
||||||
|
assert requests[-1][1] is not None
|
||||||
|
self.assertEqual(target, requests[-1][1]["target_commitish"])
|
||||||
|
|
||||||
def test_resolves_platforms_and_builds_evidence_descriptor(self) -> None:
|
def test_resolves_platforms_and_builds_evidence_descriptor(self) -> None:
|
||||||
index = {
|
index = {
|
||||||
"schemaVersion": 2,
|
"schemaVersion": 2,
|
||||||
@@ -52,6 +258,15 @@ class RuntimeDistributionBuildTests(unittest.TestCase):
|
|||||||
"registry.example/govoplan/api@sha256:" + "1" * 64,
|
"registry.example/govoplan/api@sha256:" + "1" * 64,
|
||||||
metadata["platforms"]["linux/amd64"],
|
metadata["platforms"]["linux/amd64"],
|
||||||
)
|
)
|
||||||
|
dependency_metadata = OCI.resolve_platforms(
|
||||||
|
index,
|
||||||
|
repository="registry.example:5000/library/postgres:16-alpine",
|
||||||
|
index_digest="sha256:" + "a" * 64,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"registry.example:5000/library/postgres@sha256:" + "2" * 64,
|
||||||
|
dependency_metadata["platforms"]["linux/arm64"],
|
||||||
|
)
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(prefix="govoplan-runtime-finalize-") as value:
|
with tempfile.TemporaryDirectory(prefix="govoplan-runtime-finalize-") as value:
|
||||||
root = Path(value)
|
root = Path(value)
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SCRIPT = ROOT / "tools/checks/runtime-image-smoke.py"
|
||||||
|
SPEC = importlib.util.spec_from_file_location("runtime_image_smoke", SCRIPT)
|
||||||
|
assert SPEC is not None and SPEC.loader is not None
|
||||||
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
|
sys.modules[SPEC.name] = MODULE
|
||||||
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeImageSmokeTests(unittest.TestCase):
|
||||||
|
def test_selects_the_exact_platform_digest(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value:
|
||||||
|
path = Path(value) / "metadata.json"
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"index": "registry.example/api@sha256:" + "a" * 64,
|
||||||
|
"platforms": {
|
||||||
|
"linux/amd64": "registry.example/api@sha256:" + "1" * 64,
|
||||||
|
"linux/arm64": "registry.example/api@sha256:" + "2" * 64,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
"registry.example/api@sha256:" + "2" * 64,
|
||||||
|
MODULE.platform_image(path, "linux/arm64", "API"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_rejects_mutable_or_missing_platform_images(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-runtime-smoke-") as value:
|
||||||
|
path = Path(value) / "metadata.json"
|
||||||
|
path.write_text(
|
||||||
|
json.dumps({"platforms": {"linux/amd64": "registry.example/api:latest"}}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"):
|
||||||
|
MODULE.platform_image(path, "linux/amd64", "API")
|
||||||
|
with self.assertRaisesRegex(MODULE.SmokeError, "exact sha256"):
|
||||||
|
MODULE.platform_image(path, "linux/arm64", "API")
|
||||||
|
|
||||||
|
def test_readiness_fails_immediately_when_container_exits(self) -> None:
|
||||||
|
exited = subprocess.CompletedProcess([], 0, "false\n", "")
|
||||||
|
logs = subprocess.CompletedProcess(
|
||||||
|
[], 0, "fatal startup error db-secret\n", ""
|
||||||
|
)
|
||||||
|
with patch.object(MODULE, "_run", side_effect=(exited, logs)):
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
MODULE.SmokeError,
|
||||||
|
r"container exited before readiness: fatal startup error \[redacted\]",
|
||||||
|
):
|
||||||
|
MODULE._wait_for(
|
||||||
|
"WebUI",
|
||||||
|
lambda: self.fail("probe must not run for an exited container"),
|
||||||
|
timeout=60,
|
||||||
|
container="web",
|
||||||
|
redactions=("db-secret",),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_smoke_supplies_the_packaged_web_upstream_and_schema_contract(self) -> None:
|
||||||
|
source = SCRIPT.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
self.assertIn('"--network-alias",\n "load-balancer"', source)
|
||||||
|
self.assertIn("'core_system_settings'", source)
|
||||||
|
self.assertIn("'core_runtime_nodes'", source)
|
||||||
|
self.assertIn('if platform == "linux/arm64"', source)
|
||||||
|
self.assertIn('"ARM64-COW-BUG"', source)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -25,6 +25,12 @@ from govoplan_core.core.datasources import (
|
|||||||
datasource_publication,
|
datasource_publication,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.modules import ModuleContext
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.recovery import RecoveryCheckpoint, RecoveryOperation
|
||||||
|
from govoplan_core.core.runtime_coordination import (
|
||||||
|
DistributedLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
bind_process_runtime_identity,
|
||||||
|
)
|
||||||
from govoplan_core.core.tabular_sources import (
|
from govoplan_core.core.tabular_sources import (
|
||||||
TabularSnapshotInput,
|
TabularSnapshotInput,
|
||||||
tabular_snapshot_writer,
|
tabular_snapshot_writer,
|
||||||
@@ -73,6 +79,9 @@ def main() -> int:
|
|||||||
Base.metadata.create_all(
|
Base.metadata.create_all(
|
||||||
engine,
|
engine,
|
||||||
tables=[
|
tables=[
|
||||||
|
DistributedLease.__table__,
|
||||||
|
RecoveryOperation.__table__,
|
||||||
|
RecoveryCheckpoint.__table__,
|
||||||
ConnectorTabularSource.__table__,
|
ConnectorTabularSource.__table__,
|
||||||
DatasourceRecord.__table__,
|
DatasourceRecord.__table__,
|
||||||
DatasourcePayloadRecord.__table__,
|
DatasourcePayloadRecord.__table__,
|
||||||
@@ -86,153 +95,168 @@ def main() -> int:
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
session_factory = sessionmaker(bind=engine)
|
session_factory = sessionmaker(bind=engine)
|
||||||
with session_factory() as session:
|
bind_process_runtime_identity(_runtime_identity())
|
||||||
principal = _principal()
|
try:
|
||||||
writer = tabular_snapshot_writer(registry)
|
with session_factory() as session:
|
||||||
lifecycle = datasource_lifecycle(registry)
|
principal = _principal()
|
||||||
catalogue = datasource_catalogue(registry)
|
writer = tabular_snapshot_writer(registry)
|
||||||
publisher = datasource_publication(registry)
|
lifecycle = datasource_lifecycle(registry)
|
||||||
runner = dataflow_run_lifecycle(registry)
|
catalogue = datasource_catalogue(registry)
|
||||||
if (
|
publisher = datasource_publication(registry)
|
||||||
writer is None
|
runner = dataflow_run_lifecycle(registry)
|
||||||
or lifecycle is None
|
if (
|
||||||
or catalogue is None
|
writer is None
|
||||||
or publisher is None
|
or lifecycle is None
|
||||||
or runner is None
|
or catalogue is None
|
||||||
):
|
or publisher is None
|
||||||
raise RuntimeError("Datasource composition capabilities are incomplete.")
|
or runner is None
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"Datasource composition capabilities are incomplete."
|
||||||
|
)
|
||||||
|
|
||||||
origin = writer.create_snapshot(
|
origin = writer.create_snapshot(
|
||||||
session,
|
session,
|
||||||
principal,
|
principal,
|
||||||
snapshot=TabularSnapshotInput(
|
snapshot=TabularSnapshotInput(
|
||||||
name="Monthly cases",
|
name="Monthly cases",
|
||||||
source_name="connector_monthly_cases",
|
source_name="connector_monthly_cases",
|
||||||
rows=(
|
rows=(
|
||||||
{"id": 1, "amount": 5},
|
{"id": 1, "amount": 5},
|
||||||
{"id": 2, "amount": 15},
|
{"id": 2, "amount": 15},
|
||||||
|
),
|
||||||
),
|
),
|
||||||
),
|
)
|
||||||
)
|
datasource = lifecycle.register_origin(
|
||||||
datasource = lifecycle.register_origin(
|
session,
|
||||||
session,
|
principal,
|
||||||
principal,
|
origin_ref=origin.ref,
|
||||||
origin_ref=origin.ref,
|
name="Monthly cases cache",
|
||||||
name="Monthly cases cache",
|
source_name="monthly_cases",
|
||||||
source_name="monthly_cases",
|
mode="cached",
|
||||||
mode="cached",
|
)
|
||||||
)
|
result = preview_pipeline(
|
||||||
result = preview_pipeline(
|
session,
|
||||||
session,
|
tenant_id="tenant-1",
|
||||||
tenant_id="tenant-1",
|
actor_id="account-1",
|
||||||
actor_id="account-1",
|
payload=PipelinePreviewRequest(
|
||||||
payload=PipelinePreviewRequest(
|
graph=_graph(
|
||||||
graph=_graph(
|
datasource_ref=datasource.ref,
|
||||||
datasource_ref=datasource.ref,
|
fingerprint=datasource.fingerprint,
|
||||||
fingerprint=datasource.fingerprint,
|
),
|
||||||
|
row_limit=100,
|
||||||
),
|
),
|
||||||
row_limit=100,
|
principal=principal,
|
||||||
),
|
registry=registry,
|
||||||
principal=principal,
|
)
|
||||||
registry=registry,
|
expected_rows = [
|
||||||
)
|
{"id": 1, "amount": 5},
|
||||||
expected_rows = [
|
{"id": 2, "amount": 15},
|
||||||
{"id": 1, "amount": 5},
|
]
|
||||||
{"id": 2, "amount": 15},
|
if result.status != "succeeded":
|
||||||
]
|
raise RuntimeError(
|
||||||
if result.status != "succeeded":
|
f"Dataflow preview failed: {result.diagnostics}"
|
||||||
raise RuntimeError(f"Dataflow preview failed: {result.diagnostics}")
|
)
|
||||||
if result.rows != expected_rows:
|
if result.rows != expected_rows:
|
||||||
raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
|
raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
|
||||||
if result.source_fingerprints[0]["source_ref"] != datasource.ref:
|
if result.source_fingerprints[0]["source_ref"] != datasource.ref:
|
||||||
raise RuntimeError("Dataflow lineage did not retain the datasource reference.")
|
raise RuntimeError(
|
||||||
pipeline = create_pipeline(
|
"Dataflow lineage did not retain the datasource reference."
|
||||||
session,
|
)
|
||||||
tenant_id="tenant-1",
|
pipeline = create_pipeline(
|
||||||
actor_id="account-1",
|
session,
|
||||||
payload=PipelineCreateRequest(
|
tenant_id="tenant-1",
|
||||||
name="Monthly case output",
|
actor_id="account-1",
|
||||||
status="active",
|
payload=PipelineCreateRequest(
|
||||||
graph=_graph(
|
name="Monthly case output",
|
||||||
datasource_ref=datasource.ref,
|
status="active",
|
||||||
fingerprint=datasource.fingerprint,
|
graph=_graph(
|
||||||
|
datasource_ref=datasource.ref,
|
||||||
|
fingerprint=datasource.fingerprint,
|
||||||
|
),
|
||||||
|
editor_mode="graph",
|
||||||
),
|
),
|
||||||
editor_mode="graph",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
run_request = DataflowRunRequest(
|
|
||||||
pipeline_ref=f"pipeline:{pipeline.id}",
|
|
||||||
revision=1,
|
|
||||||
idempotency_key="composition-run-1",
|
|
||||||
publication=DataflowPublicationTarget(
|
|
||||||
name="Monthly case result",
|
|
||||||
source_name="monthly_case_result",
|
|
||||||
freeze=True,
|
|
||||||
frozen_label="Composition evidence",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
published = runner.start_run(
|
|
||||||
session,
|
|
||||||
principal,
|
|
||||||
request=run_request,
|
|
||||||
)
|
|
||||||
replayed = runner.start_run(
|
|
||||||
session,
|
|
||||||
principal,
|
|
||||||
request=run_request,
|
|
||||||
)
|
|
||||||
if published.status != "queued":
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Dataflow run was not queued: {published.status}"
|
|
||||||
)
|
)
|
||||||
worker = SqlDataflowRunWorker(
|
run_request = DataflowRunRequest(
|
||||||
registry=_AutomationRegistry(registry, principal)
|
pipeline_ref=f"pipeline:{pipeline.id}",
|
||||||
)
|
revision=1,
|
||||||
worker_result = worker.dispatch_pending(
|
idempotency_key="composition-run-1",
|
||||||
session,
|
publication=DataflowPublicationTarget(
|
||||||
worker_id="composition-worker",
|
name="Monthly case result",
|
||||||
)
|
source_name="monthly_case_result",
|
||||||
if worker_result["succeeded"] != 1:
|
freeze=True,
|
||||||
raise RuntimeError(
|
frozen_label="Composition evidence",
|
||||||
f"Dataflow worker failed: {worker_result!r}"
|
),
|
||||||
)
|
)
|
||||||
completed = runner.get_run(
|
published = runner.start_run(
|
||||||
session,
|
session,
|
||||||
principal,
|
principal,
|
||||||
run_ref=published.ref,
|
request=run_request,
|
||||||
)
|
|
||||||
if completed is None:
|
|
||||||
raise RuntimeError("Dataflow run evidence disappeared.")
|
|
||||||
published = completed
|
|
||||||
if published.status != "succeeded":
|
|
||||||
raise RuntimeError(f"Dataflow publication failed: {published.error}")
|
|
||||||
if replayed.ref != published.ref or not replayed.replayed:
|
|
||||||
raise RuntimeError("Dataflow run idempotency did not replay the prior run.")
|
|
||||||
if (
|
|
||||||
not published.output_datasource_ref
|
|
||||||
or not published.output_materialization_ref
|
|
||||||
):
|
|
||||||
raise RuntimeError("Dataflow publication did not retain output references.")
|
|
||||||
output = catalogue.read_datasource(
|
|
||||||
session,
|
|
||||||
principal,
|
|
||||||
request=DatasourceReadRequest(
|
|
||||||
datasource_ref=published.output_datasource_ref,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if list(output.rows) != expected_rows:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Unexpected published Dataflow rows: {list(output.rows)!r}"
|
|
||||||
)
|
)
|
||||||
if (
|
replayed = runner.start_run(
|
||||||
output.materialization is None
|
session,
|
||||||
or output.materialization.ref != published.output_materialization_ref
|
principal,
|
||||||
or output.materialization.frozen_at is None
|
request=run_request,
|
||||||
):
|
|
||||||
raise RuntimeError(
|
|
||||||
"Published Datasource materialization is not pinned and frozen."
|
|
||||||
)
|
)
|
||||||
engine.dispose()
|
if published.status != "queued":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Dataflow run was not queued: {published.status}"
|
||||||
|
)
|
||||||
|
worker = SqlDataflowRunWorker(
|
||||||
|
registry=_AutomationRegistry(registry, principal)
|
||||||
|
)
|
||||||
|
worker_result = worker.dispatch_pending(
|
||||||
|
session,
|
||||||
|
worker_id="composition-worker",
|
||||||
|
)
|
||||||
|
if worker_result["succeeded"] != 1:
|
||||||
|
raise RuntimeError(f"Dataflow worker failed: {worker_result!r}")
|
||||||
|
completed = runner.get_run(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
run_ref=published.ref,
|
||||||
|
)
|
||||||
|
if completed is None:
|
||||||
|
raise RuntimeError("Dataflow run evidence disappeared.")
|
||||||
|
published = completed
|
||||||
|
if published.status != "succeeded":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Dataflow publication failed: {published.error}"
|
||||||
|
)
|
||||||
|
if replayed.ref != published.ref or not replayed.replayed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Dataflow run idempotency did not replay the prior run."
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not published.output_datasource_ref
|
||||||
|
or not published.output_materialization_ref
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"Dataflow publication did not retain output references."
|
||||||
|
)
|
||||||
|
output = catalogue.read_datasource(
|
||||||
|
session,
|
||||||
|
principal,
|
||||||
|
request=DatasourceReadRequest(
|
||||||
|
datasource_ref=published.output_datasource_ref,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if list(output.rows) != expected_rows:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Unexpected published Dataflow rows: {list(output.rows)!r}"
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
output.materialization is None
|
||||||
|
or output.materialization.ref
|
||||||
|
!= published.output_materialization_ref
|
||||||
|
or output.materialization.frozen_at is None
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"Published Datasource materialization is not pinned and frozen."
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
bind_process_runtime_identity(None)
|
||||||
|
engine.dispose()
|
||||||
print(
|
print(
|
||||||
"Connector -> Datasources -> pinned Dataflow publication composition passed."
|
"Connector -> Datasources -> pinned Dataflow publication composition passed."
|
||||||
)
|
)
|
||||||
@@ -252,6 +276,17 @@ class _AutomationProvider:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_identity() -> RuntimeIdentity:
|
||||||
|
return RuntimeIdentity(
|
||||||
|
installation_id="datasource-composition-check",
|
||||||
|
node_id="composition-worker",
|
||||||
|
incarnation="composition-worker-incarnation",
|
||||||
|
role="worker",
|
||||||
|
software_version="test",
|
||||||
|
composition_hash="c" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class _AutomationRegistry:
|
class _AutomationRegistry:
|
||||||
def __init__(self, registry, principal: ApiPrincipal) -> None:
|
def __init__(self, registry, principal: ApiPrincipal) -> None:
|
||||||
self.registry = registry
|
self.registry = registry
|
||||||
|
|||||||
@@ -4,13 +4,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
|
|
||||||
@@ -27,57 +28,211 @@ from govoplan_deploy.model import default_spec # noqa: E402
|
|||||||
DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
|
DIGEST_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$")
|
||||||
|
|
||||||
|
|
||||||
def _run(argv: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]:
|
def _run(
|
||||||
return subprocess.run(
|
argv: list[str],
|
||||||
argv,
|
*,
|
||||||
check=check,
|
check: bool = True,
|
||||||
capture_output=True,
|
input_text: str | None = None,
|
||||||
text=True,
|
) -> subprocess.CompletedProcess[str]:
|
||||||
timeout=60,
|
try:
|
||||||
|
return subprocess.run(
|
||||||
|
argv,
|
||||||
|
check=check,
|
||||||
|
capture_output=True,
|
||||||
|
input=input_text,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
stderr = exc.stderr.strip()
|
||||||
|
if stderr:
|
||||||
|
print(stderr, file=sys.stderr)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _write_volume_file(
|
||||||
|
*,
|
||||||
|
image: str,
|
||||||
|
volume: str,
|
||||||
|
filename: str,
|
||||||
|
content: str,
|
||||||
|
) -> None:
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+", filename):
|
||||||
|
raise ValueError(f"invalid config filename: {filename!r}")
|
||||||
|
_run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--interactive",
|
||||||
|
"--user",
|
||||||
|
"0:0",
|
||||||
|
"--mount",
|
||||||
|
f"type=volume,src={volume},dst=/govoplan-config",
|
||||||
|
"--entrypoint",
|
||||||
|
"sh",
|
||||||
|
image,
|
||||||
|
"-c",
|
||||||
|
f"umask 022; cat > /govoplan-config/{filename}",
|
||||||
|
],
|
||||||
|
input_text=content,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _published_port(container: str, target: int) -> int:
|
def _published_port(container: str, target: int) -> int:
|
||||||
output = _run(["docker", "port", container, f"{target}/tcp"]).stdout.strip()
|
output = _run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"inspect",
|
||||||
|
"--format",
|
||||||
|
"{{json .HostConfig.PortBindings}}",
|
||||||
|
container,
|
||||||
|
]
|
||||||
|
).stdout.strip()
|
||||||
try:
|
try:
|
||||||
return int(output.rsplit(":", 1)[1])
|
bindings = json.loads(output)[f"{target}/tcp"]
|
||||||
except (IndexError, ValueError) as exc:
|
if not isinstance(bindings, list) or len(bindings) != 1:
|
||||||
raise RuntimeError(f"cannot determine published port from {output!r}") from exc
|
raise ValueError("expected exactly one published binding")
|
||||||
|
binding = bindings[0]
|
||||||
|
if binding.get("HostIp") != "127.0.0.1":
|
||||||
|
raise ValueError("published binding is not loopback-only")
|
||||||
|
return int(binding["HostPort"])
|
||||||
|
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"cannot determine loopback binding for {target}/tcp from {output!r}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
def _curl(url: str, *, headers: bool = False) -> str:
|
def _available_loopback_port(*, exclude: frozenset[int] = frozenset()) -> int:
|
||||||
argv = ["curl", "--silent", "--show-error", "--insecure"]
|
for _attempt in range(10):
|
||||||
if headers:
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
|
||||||
argv.extend(["--head"])
|
listener.bind(("127.0.0.1", 0))
|
||||||
argv.append(url)
|
port = int(listener.getsockname()[1])
|
||||||
return _run(argv).stdout
|
if port not in exclude:
|
||||||
|
return port
|
||||||
|
raise RuntimeError("cannot allocate distinct loopback ports for ingress drill")
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_https(port: int) -> str:
|
def _probe_ingress(*, image: str, network: str, container: str) -> None:
|
||||||
deadline = time.monotonic() + 30
|
probe = r'''
|
||||||
last_error = ""
|
import socket
|
||||||
while time.monotonic() < deadline:
|
import ssl
|
||||||
try:
|
import time
|
||||||
return _curl(f"https://localhost:{port}/health")
|
|
||||||
except subprocess.CalledProcessError as exc:
|
|
||||||
last_error = exc.stderr.strip()
|
def request(port, payload, *, tls):
|
||||||
time.sleep(0.5)
|
connection = socket.create_connection(("ingress", port), timeout=3)
|
||||||
raise RuntimeError(f"managed ingress did not become ready: {last_error}")
|
if tls:
|
||||||
|
connection = ssl._create_unverified_context().wrap_socket(
|
||||||
|
connection, server_hostname="localhost"
|
||||||
|
)
|
||||||
|
with connection:
|
||||||
|
connection.sendall(payload)
|
||||||
|
chunks = []
|
||||||
|
while True:
|
||||||
|
chunk = connection.recv(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
chunks.append(chunk)
|
||||||
|
return b"".join(chunks)
|
||||||
|
|
||||||
|
|
||||||
|
deadline = time.monotonic() + 30
|
||||||
|
last_error = ""
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
response = request(
|
||||||
|
8443,
|
||||||
|
b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
|
||||||
|
tls=True,
|
||||||
|
)
|
||||||
|
head, body_bytes = response.split(b"\r\n\r\n", 1)
|
||||||
|
status = int(head.split(b" ", 2)[1])
|
||||||
|
if status != 200:
|
||||||
|
raise RuntimeError(f"HTTPS returned {status}, expected 200")
|
||||||
|
body = body_bytes.decode("utf-8").strip()
|
||||||
|
if body != "proto=https":
|
||||||
|
raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
|
||||||
|
break
|
||||||
|
except Exception as exc:
|
||||||
|
last_error = f"{type(exc).__name__}: {exc}"
|
||||||
|
time.sleep(0.5)
|
||||||
|
else:
|
||||||
|
raise SystemExit(f"managed ingress did not become ready: {last_error}")
|
||||||
|
|
||||||
|
response = request(
|
||||||
|
8080,
|
||||||
|
b"HEAD /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n",
|
||||||
|
tls=False,
|
||||||
|
)
|
||||||
|
head = response.split(b"\r\n\r\n", 1)[0].decode("iso-8859-1")
|
||||||
|
lines = head.split("\r\n")
|
||||||
|
status = int(lines[0].split(" ", 2)[1])
|
||||||
|
if status != 308:
|
||||||
|
raise SystemExit(f"HTTP returned {status}, expected redirect 308")
|
||||||
|
headers = {
|
||||||
|
key.lower(): value.strip()
|
||||||
|
for key, separator, value in (line.partition(":") for line in lines[1:])
|
||||||
|
if separator
|
||||||
|
}
|
||||||
|
location = headers.get("location", "")
|
||||||
|
if not location.startswith("https://localhost"):
|
||||||
|
raise SystemExit(f"HTTP redirect had unexpected location: {location!r}")
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
_run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--network",
|
||||||
|
network,
|
||||||
|
"--read-only",
|
||||||
|
"--security-opt",
|
||||||
|
"no-new-privileges",
|
||||||
|
"--cap-drop",
|
||||||
|
"ALL",
|
||||||
|
"--entrypoint",
|
||||||
|
"python",
|
||||||
|
image,
|
||||||
|
"-c",
|
||||||
|
probe,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
_print_container_diagnostics(container)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _print_container_diagnostics(container: str) -> None:
|
||||||
|
state = _run(
|
||||||
|
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
state_detail = (state.stdout + state.stderr).strip()
|
||||||
|
if state_detail:
|
||||||
|
print(f"managed ingress state:\n{state_detail}", file=sys.stderr)
|
||||||
|
logs = _run(["docker", "logs", container], check=False)
|
||||||
|
log_detail = (logs.stdout + logs.stderr).strip()
|
||||||
|
if log_detail:
|
||||||
|
print(f"managed ingress logs:\n{log_detail}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--caddy-image", required=True)
|
parser.add_argument("--caddy-image", required=True)
|
||||||
parser.add_argument("--load-balancer-image", required=True)
|
parser.add_argument("--load-balancer-image", required=True)
|
||||||
|
parser.add_argument("--probe-image", required=True)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
for label, image in (
|
for label, image in (
|
||||||
("--caddy-image", args.caddy_image),
|
("--caddy-image", args.caddy_image),
|
||||||
("--load-balancer-image", args.load_balancer_image),
|
("--load-balancer-image", args.load_balancer_image),
|
||||||
|
("--probe-image", args.probe_image),
|
||||||
):
|
):
|
||||||
if DIGEST_IMAGE.fullmatch(image) is None:
|
if DIGEST_IMAGE.fullmatch(image) is None:
|
||||||
parser.error(f"{label} must be pinned by sha256 digest")
|
parser.error(f"{label} must be pinned by sha256 digest")
|
||||||
if shutil.which("docker") is None or shutil.which("curl") is None:
|
if shutil.which("docker") is None:
|
||||||
parser.error("docker and curl are required")
|
parser.error("docker is required")
|
||||||
|
|
||||||
suffix = uuid4().hex[:10]
|
suffix = uuid4().hex[:10]
|
||||||
network = f"govoplan-ingress-drill-{suffix}"
|
network = f"govoplan-ingress-drill-{suffix}"
|
||||||
@@ -85,15 +240,33 @@ def main() -> int:
|
|||||||
backend = f"govoplan-ingress-backend-{suffix}"
|
backend = f"govoplan-ingress-backend-{suffix}"
|
||||||
data_volume = f"govoplan-ingress-data-{suffix}"
|
data_volume = f"govoplan-ingress-data-{suffix}"
|
||||||
config_volume = f"govoplan-ingress-config-{suffix}"
|
config_volume = f"govoplan-ingress-config-{suffix}"
|
||||||
|
backend_config_volume = f"govoplan-ingress-backend-config-{suffix}"
|
||||||
|
ingress_config_volume = f"govoplan-ingress-caddy-config-{suffix}"
|
||||||
|
load_balancer_config_volume = f"govoplan-ingress-haproxy-config-{suffix}"
|
||||||
cleanup = [
|
cleanup = [
|
||||||
["docker", "rm", "--force", ingress, backend],
|
["docker", "rm", "--force", ingress, backend],
|
||||||
["docker", "network", "rm", network],
|
["docker", "network", "rm", network],
|
||||||
["docker", "volume", "rm", data_volume, config_volume],
|
[
|
||||||
|
"docker",
|
||||||
|
"volume",
|
||||||
|
"rm",
|
||||||
|
data_volume,
|
||||||
|
config_volume,
|
||||||
|
backend_config_volume,
|
||||||
|
ingress_config_volume,
|
||||||
|
load_balancer_config_volume,
|
||||||
|
],
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
_run(["docker", "network", "create", network])
|
_run(["docker", "network", "create", network])
|
||||||
_run(["docker", "volume", "create", data_volume])
|
for volume in (
|
||||||
_run(["docker", "volume", "create", config_volume])
|
data_volume,
|
||||||
|
config_volume,
|
||||||
|
backend_config_volume,
|
||||||
|
ingress_config_volume,
|
||||||
|
load_balancer_config_volume,
|
||||||
|
):
|
||||||
|
_run(["docker", "volume", "create", volume])
|
||||||
with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory:
|
with tempfile.TemporaryDirectory(prefix="govoplan-ingress-") as directory:
|
||||||
root = Path(directory)
|
root = Path(directory)
|
||||||
backend_config = root / "backend.Caddyfile"
|
backend_config = root / "backend.Caddyfile"
|
||||||
@@ -127,6 +300,24 @@ def main() -> int:
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
load_balancer_config.chmod(0o644)
|
load_balancer_config.chmod(0o644)
|
||||||
|
_write_volume_file(
|
||||||
|
image=args.load_balancer_image,
|
||||||
|
volume=load_balancer_config_volume,
|
||||||
|
filename="haproxy.cfg",
|
||||||
|
content=load_balancer_config.read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
|
_write_volume_file(
|
||||||
|
image=args.caddy_image,
|
||||||
|
volume=backend_config_volume,
|
||||||
|
filename="Caddyfile",
|
||||||
|
content=backend_config.read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
|
_write_volume_file(
|
||||||
|
image=args.caddy_image,
|
||||||
|
volume=ingress_config_volume,
|
||||||
|
filename="Caddyfile",
|
||||||
|
content=ingress_config.read_text(encoding="utf-8"),
|
||||||
|
)
|
||||||
_run(
|
_run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
@@ -136,7 +327,11 @@ def main() -> int:
|
|||||||
"--cap-drop",
|
"--cap-drop",
|
||||||
"ALL",
|
"ALL",
|
||||||
"--mount",
|
"--mount",
|
||||||
f"type=bind,src={load_balancer_config},dst=/usr/local/etc/haproxy/haproxy.cfg,readonly",
|
(
|
||||||
|
"type=volume,"
|
||||||
|
f"src={load_balancer_config_volume},"
|
||||||
|
"dst=/usr/local/etc/haproxy,readonly"
|
||||||
|
),
|
||||||
args.load_balancer_image,
|
args.load_balancer_image,
|
||||||
"haproxy",
|
"haproxy",
|
||||||
"-c",
|
"-c",
|
||||||
@@ -154,18 +349,28 @@ def main() -> int:
|
|||||||
backend,
|
backend,
|
||||||
"--network",
|
"--network",
|
||||||
network,
|
network,
|
||||||
|
"--network-alias",
|
||||||
|
"load-balancer",
|
||||||
"--read-only",
|
"--read-only",
|
||||||
"--tmpfs",
|
"--tmpfs",
|
||||||
"/tmp:rw,noexec,nosuid,size=16m",
|
"/tmp:rw,noexec,nosuid,size=16m",
|
||||||
"--mount",
|
"--mount",
|
||||||
f"type=bind,src={backend_config},dst=/etc/caddy/Caddyfile,readonly",
|
(
|
||||||
|
"type=volume,"
|
||||||
|
f"src={backend_config_volume},"
|
||||||
|
"dst=/govoplan-config,readonly"
|
||||||
|
),
|
||||||
args.caddy_image,
|
args.caddy_image,
|
||||||
"caddy",
|
"caddy",
|
||||||
"run",
|
"run",
|
||||||
"--config",
|
"--config",
|
||||||
"/etc/caddy/Caddyfile",
|
"/govoplan-config/Caddyfile",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
requested_http_port = _available_loopback_port()
|
||||||
|
requested_https_port = _available_loopback_port(
|
||||||
|
exclude=frozenset({requested_http_port})
|
||||||
|
)
|
||||||
ingress_command = [
|
ingress_command = [
|
||||||
"docker",
|
"docker",
|
||||||
"run",
|
"run",
|
||||||
@@ -174,6 +379,8 @@ def main() -> int:
|
|||||||
ingress,
|
ingress,
|
||||||
"--network",
|
"--network",
|
||||||
network,
|
network,
|
||||||
|
"--network-alias",
|
||||||
|
"ingress",
|
||||||
"--read-only",
|
"--read-only",
|
||||||
"--tmpfs",
|
"--tmpfs",
|
||||||
"/tmp:rw,noexec,nosuid,size=16m",
|
"/tmp:rw,noexec,nosuid,size=16m",
|
||||||
@@ -181,12 +388,18 @@ def main() -> int:
|
|||||||
"no-new-privileges",
|
"no-new-privileges",
|
||||||
"--cap-drop",
|
"--cap-drop",
|
||||||
"ALL",
|
"ALL",
|
||||||
|
"--cap-add",
|
||||||
|
"NET_BIND_SERVICE",
|
||||||
"--publish",
|
"--publish",
|
||||||
"127.0.0.1::8080",
|
f"127.0.0.1:{requested_http_port}:8080/tcp",
|
||||||
"--publish",
|
"--publish",
|
||||||
"127.0.0.1::8443",
|
f"127.0.0.1:{requested_https_port}:8443/tcp",
|
||||||
"--mount",
|
"--mount",
|
||||||
f"type=bind,src={ingress_config},dst=/etc/caddy/Caddyfile,readonly",
|
(
|
||||||
|
"type=volume,"
|
||||||
|
f"src={ingress_config_volume},"
|
||||||
|
"dst=/govoplan-config,readonly"
|
||||||
|
),
|
||||||
"--mount",
|
"--mount",
|
||||||
f"type=volume,src={data_volume},dst=/data",
|
f"type=volume,src={data_volume},dst=/data",
|
||||||
"--mount",
|
"--mount",
|
||||||
@@ -195,25 +408,32 @@ def main() -> int:
|
|||||||
"caddy",
|
"caddy",
|
||||||
"run",
|
"run",
|
||||||
"--config",
|
"--config",
|
||||||
"/etc/caddy/Caddyfile",
|
"/govoplan-config/Caddyfile",
|
||||||
]
|
]
|
||||||
_run(ingress_command)
|
_run(ingress_command)
|
||||||
http_port = _published_port(ingress, 8080)
|
http_port = _published_port(ingress, 8080)
|
||||||
https_port = _published_port(ingress, 8443)
|
https_port = _published_port(ingress, 8443)
|
||||||
body = _wait_for_https(https_port)
|
if (http_port, https_port) != (
|
||||||
if body.strip() != "proto=https":
|
requested_http_port,
|
||||||
raise RuntimeError(f"forwarded protocol was not normalized: {body!r}")
|
requested_https_port,
|
||||||
redirect = _curl(f"http://localhost:{http_port}/health", headers=True)
|
):
|
||||||
if not redirect.startswith("HTTP/1.1 308"):
|
raise RuntimeError("Docker published unexpected ingress ports")
|
||||||
raise RuntimeError(f"HTTP was not redirected to HTTPS: {redirect!r}")
|
_probe_ingress(
|
||||||
|
image=args.probe_image,
|
||||||
|
network=network,
|
||||||
|
container=ingress,
|
||||||
|
)
|
||||||
|
|
||||||
_run(["docker", "rm", "--force", ingress])
|
_run(["docker", "rm", "--force", ingress])
|
||||||
_run(ingress_command)
|
_run(ingress_command)
|
||||||
https_port = _published_port(ingress, 8443)
|
https_port = _published_port(ingress, 8443)
|
||||||
if _wait_for_https(https_port).strip() != "proto=https":
|
if https_port != requested_https_port:
|
||||||
raise RuntimeError(
|
raise RuntimeError("Docker changed the ingress TLS binding on restart")
|
||||||
"managed ingress did not recover with persistent state"
|
_probe_ingress(
|
||||||
)
|
image=args.probe_image,
|
||||||
|
network=network,
|
||||||
|
container=ingress,
|
||||||
|
)
|
||||||
_run(
|
_run(
|
||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
|
|||||||
@@ -0,0 +1,656 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Exercise a pinned GovOPlaN runtime image pair on one OCI platform."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from typing import Callable, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
PLATFORMS = frozenset({"linux/amd64", "linux/arm64"})
|
||||||
|
DIGEST_IMAGE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$")
|
||||||
|
BASE_MODULES = (
|
||||||
|
"tenancy",
|
||||||
|
"organizations",
|
||||||
|
"identity",
|
||||||
|
"idm",
|
||||||
|
"access",
|
||||||
|
"admin",
|
||||||
|
"dashboard",
|
||||||
|
"policy",
|
||||||
|
"audit",
|
||||||
|
"docs",
|
||||||
|
"ops",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SmokeError(RuntimeError):
|
||||||
|
"""A runtime image failed its bounded acceptance drill."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--api-metadata", type=Path, required=True)
|
||||||
|
parser.add_argument("--web-metadata", type=Path, required=True)
|
||||||
|
parser.add_argument("--postgres-metadata", type=Path, required=True)
|
||||||
|
parser.add_argument("--redis-metadata", type=Path, required=True)
|
||||||
|
parser.add_argument("--platform", choices=sorted(PLATFORMS), required=True)
|
||||||
|
parser.add_argument("--output", type=Path, required=True)
|
||||||
|
parser.add_argument("--timeout-seconds", type=float, default=600.0)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> str:
|
||||||
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
def _digest_image(value: object, label: str) -> str:
|
||||||
|
if not isinstance(value, str) or DIGEST_IMAGE.fullmatch(value) is None:
|
||||||
|
raise SmokeError(f"{label} must be an exact sha256 image reference")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def platform_image(path: Path, platform: str, label: str) -> str:
|
||||||
|
try:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise SmokeError(f"cannot read {label} OCI metadata") from exc
|
||||||
|
if not isinstance(payload, dict) or not isinstance(payload.get("platforms"), dict):
|
||||||
|
raise SmokeError(f"{label} OCI metadata has no platform map")
|
||||||
|
return _digest_image(payload["platforms"].get(platform), f"{label} {platform}")
|
||||||
|
|
||||||
|
|
||||||
|
def _tail(value: str, *, limit: int = 4000) -> str:
|
||||||
|
return value[-limit:].strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
arguments: Sequence[str],
|
||||||
|
*,
|
||||||
|
check: bool = True,
|
||||||
|
timeout: float = 600.0,
|
||||||
|
redactions: Sequence[str] = (),
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
list(arguments),
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise SmokeError(f"container command could not complete: {type(exc).__name__}") from exc
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
detail = _tail(result.stderr or result.stdout or "no diagnostic output")
|
||||||
|
for secret in redactions:
|
||||||
|
if secret:
|
||||||
|
detail = detail.replace(secret, "[redacted]")
|
||||||
|
raise SmokeError(f"container command failed: {detail}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for(
|
||||||
|
label: str,
|
||||||
|
probe: Callable[[], subprocess.CompletedProcess[str]],
|
||||||
|
*,
|
||||||
|
timeout: float,
|
||||||
|
container: str | None = None,
|
||||||
|
redactions: Sequence[str] = (),
|
||||||
|
) -> None:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
last = ""
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if container is not None:
|
||||||
|
state = _run(
|
||||||
|
("docker", "inspect", "--format", "{{.State.Running}}", container),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if state.returncode != 0 or state.stdout.strip() != "true":
|
||||||
|
logs = _run(
|
||||||
|
("docker", "logs", "--tail", "100", container),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
detail = _tail(logs.stdout + logs.stderr, limit=8000)
|
||||||
|
for secret in redactions:
|
||||||
|
if secret:
|
||||||
|
detail = detail.replace(secret, "[redacted]")
|
||||||
|
raise SmokeError(
|
||||||
|
f"{label} container exited before readiness: "
|
||||||
|
f"{detail or 'no diagnostic output'}"
|
||||||
|
)
|
||||||
|
result = probe()
|
||||||
|
if result.returncode == 0:
|
||||||
|
return
|
||||||
|
last = _tail(result.stderr or result.stdout)
|
||||||
|
time.sleep(2.0)
|
||||||
|
raise SmokeError(f"{label} did not become ready: {last or 'probe failed'}")
|
||||||
|
|
||||||
|
|
||||||
|
def _environment(
|
||||||
|
*,
|
||||||
|
database_password: str,
|
||||||
|
master_key: str,
|
||||||
|
platform_slug: str,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
database = (
|
||||||
|
f"postgresql+psycopg://govoplan:{database_password}@postgres:5432/govoplan"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"APP_ENV": "dev",
|
||||||
|
"GOVOPLAN_INSTALL_PROFILE": "evaluation",
|
||||||
|
"GOVOPLAN_INSTALLATION_ID": f"runtime-smoke-{platform_slug}",
|
||||||
|
"GOVOPLAN_STATE_PROFILE": "host-shared",
|
||||||
|
"GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS": "5",
|
||||||
|
"GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS": "30",
|
||||||
|
"GOVOPLAN_EXPECTED_API_REPLICAS": "1",
|
||||||
|
"GOVOPLAN_EXPECTED_WORKER_REPLICAS": "1",
|
||||||
|
"DATABASE_URL": database,
|
||||||
|
"GOVOPLAN_DATABASE_URL_PGTOOLS": (
|
||||||
|
f"postgresql://govoplan:{database_password}@postgres:5432/govoplan"
|
||||||
|
),
|
||||||
|
"GOVOPLAN_DB_CONNECTION_LIMIT": "100",
|
||||||
|
"GOVOPLAN_DB_CONNECTION_RESERVE": "10",
|
||||||
|
"REDIS_URL": "redis://redis:6379/0",
|
||||||
|
"CELERY_ENABLED": "true",
|
||||||
|
"CELERY_QUEUES": "default",
|
||||||
|
"CELERY_WORKER_CONCURRENCY": "1",
|
||||||
|
"ENABLED_MODULES": ",".join(BASE_MODULES),
|
||||||
|
"GOVOPLAN_MIGRATION_TRACK": "release",
|
||||||
|
"DEV_AUTO_MIGRATE_ENABLED": "false",
|
||||||
|
"DEV_BOOTSTRAP_ENABLED": "false",
|
||||||
|
"AUTH_LOGIN_THROTTLE_ENABLED": "true",
|
||||||
|
"AUTH_COOKIE_SECURE": "false",
|
||||||
|
"CORS_ORIGINS": "http://localhost",
|
||||||
|
"GOVOPLAN_TRUSTED_HOSTS": "127.0.0.1,localhost,api",
|
||||||
|
"FORWARDED_ALLOW_IPS": "127.0.0.1",
|
||||||
|
"MASTER_KEY_B64": master_key,
|
||||||
|
"FILE_STORAGE_BACKEND": "local",
|
||||||
|
"FILE_STORAGE_LOCAL_ROOT": "/var/lib/govoplan/files",
|
||||||
|
"GOVOPLAN_MODULE_LIVE_APPLY_ENABLED": "false",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _env_arguments(values: dict[str, str], *, role: str, node_id: str) -> list[str]:
|
||||||
|
arguments: list[str] = []
|
||||||
|
for key, value in sorted(
|
||||||
|
{**values, "GOVOPLAN_RUNTIME_ROLE": role, "GOVOPLAN_NODE_ID": node_id}.items()
|
||||||
|
):
|
||||||
|
arguments.extend(("--env", f"{key}={value}"))
|
||||||
|
return arguments
|
||||||
|
|
||||||
|
|
||||||
|
def run_smoke(
|
||||||
|
*,
|
||||||
|
api_image: str,
|
||||||
|
web_image: str,
|
||||||
|
postgres_image: str,
|
||||||
|
redis_image: str,
|
||||||
|
platform: str,
|
||||||
|
timeout: float,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
for label, value in (
|
||||||
|
("API image", api_image),
|
||||||
|
("Web image", web_image),
|
||||||
|
("PostgreSQL image", postgres_image),
|
||||||
|
("Redis image", redis_image),
|
||||||
|
):
|
||||||
|
_digest_image(value, label)
|
||||||
|
if platform not in PLATFORMS:
|
||||||
|
raise SmokeError(f"unsupported runtime smoke platform: {platform}")
|
||||||
|
|
||||||
|
slug = platform.replace("linux/", "").replace("/", "-")
|
||||||
|
suffix = secrets.token_hex(4)
|
||||||
|
prefix = f"govoplan-runtime-{slug}-{suffix}"
|
||||||
|
names = {
|
||||||
|
"network": f"{prefix}-network",
|
||||||
|
"volume": f"{prefix}-data",
|
||||||
|
"postgres": f"{prefix}-postgres",
|
||||||
|
"redis": f"{prefix}-redis",
|
||||||
|
"api": f"{prefix}-api",
|
||||||
|
"web": f"{prefix}-web",
|
||||||
|
"worker": f"{prefix}-worker",
|
||||||
|
}
|
||||||
|
database_password = secrets.token_hex(20)
|
||||||
|
master_key = base64.urlsafe_b64encode(os.urandom(32)).decode("ascii")
|
||||||
|
redactions = (database_password, master_key)
|
||||||
|
environment = _environment(
|
||||||
|
database_password=database_password,
|
||||||
|
master_key=master_key,
|
||||||
|
platform_slug=slug,
|
||||||
|
)
|
||||||
|
checks: list[dict[str, object]] = []
|
||||||
|
started = time.monotonic()
|
||||||
|
|
||||||
|
def record(check_id: str, began: float) -> None:
|
||||||
|
duration = round(time.monotonic() - began, 3)
|
||||||
|
checks.append(
|
||||||
|
{
|
||||||
|
"id": check_id,
|
||||||
|
"state": "passed",
|
||||||
|
"duration_seconds": duration,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
print(f"PASS {platform} {check_id} ({duration}s)", flush=True)
|
||||||
|
|
||||||
|
common_runtime = [
|
||||||
|
"--platform",
|
||||||
|
platform,
|
||||||
|
"--network",
|
||||||
|
names["network"],
|
||||||
|
"--read-only",
|
||||||
|
"--tmpfs",
|
||||||
|
"/tmp:rw,noexec,nosuid,size=64m",
|
||||||
|
"--security-opt",
|
||||||
|
"no-new-privileges:true",
|
||||||
|
"--cap-drop",
|
||||||
|
"ALL",
|
||||||
|
"--mount",
|
||||||
|
f"type=volume,source={names['volume']},target=/var/lib/govoplan",
|
||||||
|
]
|
||||||
|
redis_command = ["redis-server", "--save", "", "--appendonly", "no"]
|
||||||
|
if platform == "linux/arm64":
|
||||||
|
# QEMU user-mode execution triggers Redis's host-kernel COW guard even
|
||||||
|
# though this isolated smoke disables every persistence mechanism.
|
||||||
|
redis_command.extend(("--ignore-warnings", "ARM64-COW-BUG"))
|
||||||
|
|
||||||
|
try:
|
||||||
|
_run(("docker", "network", "create", names["network"]), timeout=timeout)
|
||||||
|
_run(("docker", "volume", "create", names["volume"]), timeout=timeout)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--platform",
|
||||||
|
platform,
|
||||||
|
"--name",
|
||||||
|
names["postgres"],
|
||||||
|
"--network",
|
||||||
|
names["network"],
|
||||||
|
"--network-alias",
|
||||||
|
"postgres",
|
||||||
|
"--env",
|
||||||
|
"POSTGRES_DB=govoplan",
|
||||||
|
"--env",
|
||||||
|
"POSTGRES_USER=govoplan",
|
||||||
|
"--env",
|
||||||
|
f"POSTGRES_PASSWORD={database_password}",
|
||||||
|
"--tmpfs",
|
||||||
|
"/var/lib/postgresql/data:rw,noexec,nosuid,size=384m",
|
||||||
|
postgres_image,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--platform",
|
||||||
|
platform,
|
||||||
|
"--name",
|
||||||
|
names["redis"],
|
||||||
|
"--network",
|
||||||
|
names["network"],
|
||||||
|
"--network-alias",
|
||||||
|
"redis",
|
||||||
|
"--read-only",
|
||||||
|
"--tmpfs",
|
||||||
|
"/data:rw,noexec,nosuid,size=64m",
|
||||||
|
redis_image,
|
||||||
|
*redis_command,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
_wait_for(
|
||||||
|
"PostgreSQL",
|
||||||
|
lambda: _run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
names["postgres"],
|
||||||
|
"pg_isready",
|
||||||
|
"--username",
|
||||||
|
"govoplan",
|
||||||
|
"--dbname",
|
||||||
|
"govoplan",
|
||||||
|
),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
container=names["postgres"],
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_wait_for(
|
||||||
|
"Redis",
|
||||||
|
lambda: _run(
|
||||||
|
("docker", "exec", names["redis"], "redis-cli", "ping"),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
container=names["redis"],
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
record("managed_dependencies_ready", began)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--name",
|
||||||
|
f"{prefix}-migrate",
|
||||||
|
*common_runtime,
|
||||||
|
*_env_arguments(
|
||||||
|
environment,
|
||||||
|
role="migration",
|
||||||
|
node_id=f"runtime-smoke-{slug}-migration",
|
||||||
|
),
|
||||||
|
api_image,
|
||||||
|
"python",
|
||||||
|
"-m",
|
||||||
|
"govoplan_core.commands.init_db",
|
||||||
|
"--migration-track",
|
||||||
|
"release",
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
record("release_migrations", began)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--name",
|
||||||
|
f"{prefix}-schema",
|
||||||
|
*common_runtime,
|
||||||
|
*_env_arguments(
|
||||||
|
environment,
|
||||||
|
role="migration",
|
||||||
|
node_id=f"runtime-smoke-{slug}-schema",
|
||||||
|
),
|
||||||
|
api_image,
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
(
|
||||||
|
"import os;"
|
||||||
|
"from sqlalchemy import create_engine,inspect;"
|
||||||
|
"engine=create_engine(os.environ['DATABASE_URL']);"
|
||||||
|
"tables=set(inspect(engine).get_table_names());"
|
||||||
|
"required={'alembic_version','core_scopes','core_system_settings',"
|
||||||
|
"'core_runtime_nodes'};"
|
||||||
|
"missing=required-tables;"
|
||||||
|
"assert not missing, f'missing release tables: {sorted(missing)}';"
|
||||||
|
"engine.dispose()"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
record("release_schema_contract", began)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--name",
|
||||||
|
names["api"],
|
||||||
|
"--network-alias",
|
||||||
|
"api",
|
||||||
|
"--network-alias",
|
||||||
|
"load-balancer",
|
||||||
|
*common_runtime,
|
||||||
|
*_env_arguments(
|
||||||
|
environment,
|
||||||
|
role="api",
|
||||||
|
node_id=f"runtime-smoke-{slug}-api",
|
||||||
|
),
|
||||||
|
api_image,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_wait_for(
|
||||||
|
"GovOPlaN API",
|
||||||
|
lambda: _run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
names["api"],
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
(
|
||||||
|
"import urllib.request;"
|
||||||
|
"r=urllib.request.Request('http://127.0.0.1:8000/health/ready',"
|
||||||
|
"headers={'Host':'127.0.0.1'});"
|
||||||
|
"assert urllib.request.urlopen(r,timeout=3).status==200"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
container=names["api"],
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
names["api"],
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import os; assert os.getuid() == 10001",
|
||||||
|
),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
record("api_non_root_readiness", began)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--platform",
|
||||||
|
platform,
|
||||||
|
"--name",
|
||||||
|
names["web"],
|
||||||
|
"--network",
|
||||||
|
names["network"],
|
||||||
|
"--network-alias",
|
||||||
|
"web",
|
||||||
|
"--read-only",
|
||||||
|
"--tmpfs",
|
||||||
|
"/tmp:rw,noexec,nosuid,size=64m",
|
||||||
|
"--security-opt",
|
||||||
|
"no-new-privileges:true",
|
||||||
|
"--cap-drop",
|
||||||
|
"ALL",
|
||||||
|
web_image,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
_wait_for(
|
||||||
|
"GovOPlaN WebUI",
|
||||||
|
lambda: _run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
names["api"],
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
(
|
||||||
|
"import urllib.request;"
|
||||||
|
"assert urllib.request.urlopen('http://web:8080/health',timeout=3).status==200;"
|
||||||
|
"assert urllib.request.urlopen('http://web:8080/',timeout=3).status==200"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
container=names["web"],
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_run(
|
||||||
|
("docker", "exec", names["web"], "sh", "-c", "test \"$(id -u)\" = 101"),
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
record("web_non_root_readiness", began)
|
||||||
|
|
||||||
|
began = time.monotonic()
|
||||||
|
_run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--name",
|
||||||
|
names["worker"],
|
||||||
|
*common_runtime,
|
||||||
|
*_env_arguments(
|
||||||
|
environment,
|
||||||
|
role="worker",
|
||||||
|
node_id=f"runtime-smoke-{slug}-worker",
|
||||||
|
),
|
||||||
|
api_image,
|
||||||
|
"python",
|
||||||
|
"-m",
|
||||||
|
"celery",
|
||||||
|
"-A",
|
||||||
|
"govoplan_core.celery_app:celery",
|
||||||
|
"worker",
|
||||||
|
"--queues",
|
||||||
|
"default",
|
||||||
|
"--pool",
|
||||||
|
"solo",
|
||||||
|
"--concurrency",
|
||||||
|
"1",
|
||||||
|
"--hostname",
|
||||||
|
f"runtime-smoke-{slug}@%h",
|
||||||
|
"--loglevel",
|
||||||
|
"WARNING",
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_wait_for(
|
||||||
|
"GovOPlaN worker",
|
||||||
|
lambda: _run(
|
||||||
|
(
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
names["api"],
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
(
|
||||||
|
"from govoplan_core.celery_app import celery;"
|
||||||
|
"result=celery.send_task('govoplan.ping',queue='default');"
|
||||||
|
"assert result.get(timeout=10)=='pong'"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
),
|
||||||
|
timeout=timeout,
|
||||||
|
container=names["worker"],
|
||||||
|
redactions=redactions,
|
||||||
|
)
|
||||||
|
_run(("docker", "stop", "--time", "20", names["worker"]), timeout=30)
|
||||||
|
record("worker_delivery_and_shutdown", began)
|
||||||
|
except SmokeError as exc:
|
||||||
|
for role in ("api", "web", "worker", "postgres", "redis"):
|
||||||
|
result = _run(
|
||||||
|
("docker", "logs", "--tail", "100", names[role]),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if result.stdout or result.stderr:
|
||||||
|
detail = _tail(result.stdout + result.stderr, limit=8000)
|
||||||
|
for secret in redactions:
|
||||||
|
detail = detail.replace(secret, "[redacted]")
|
||||||
|
print(f"--- {role} logs ---\n{detail}")
|
||||||
|
raise exc
|
||||||
|
finally:
|
||||||
|
for role in ("worker", "web", "api", "redis", "postgres"):
|
||||||
|
_run(
|
||||||
|
("docker", "rm", "--force", names[role]),
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
_run(("docker", "volume", "rm", "--force", names["volume"]), check=False)
|
||||||
|
_run(("docker", "network", "rm", names["network"]), check=False)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"schema_version": "1",
|
||||||
|
"evidence_kind": "govoplan.runtime-image-smoke",
|
||||||
|
"captured_at": _utc_now(),
|
||||||
|
"platform": platform,
|
||||||
|
"images": {
|
||||||
|
"api": api_image,
|
||||||
|
"web": web_image,
|
||||||
|
"postgres": postgres_image,
|
||||||
|
"redis": redis_image,
|
||||||
|
},
|
||||||
|
"result": {"state": "passed"},
|
||||||
|
"checks": checks,
|
||||||
|
"duration_seconds": round(time.monotonic() - started, 3),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
try:
|
||||||
|
api_image = platform_image(args.api_metadata, args.platform, "API")
|
||||||
|
web_image = platform_image(args.web_metadata, args.platform, "Web")
|
||||||
|
postgres_image = platform_image(
|
||||||
|
args.postgres_metadata,
|
||||||
|
args.platform,
|
||||||
|
"PostgreSQL",
|
||||||
|
)
|
||||||
|
redis_image = platform_image(args.redis_metadata, args.platform, "Redis")
|
||||||
|
evidence = run_smoke(
|
||||||
|
api_image=api_image,
|
||||||
|
web_image=web_image,
|
||||||
|
postgres_image=postgres_image,
|
||||||
|
redis_image=redis_image,
|
||||||
|
platform=args.platform,
|
||||||
|
timeout=args.timeout_seconds,
|
||||||
|
)
|
||||||
|
except (OSError, SmokeError, ValueError) as exc:
|
||||||
|
print(f"runtime image smoke failed: {exc}")
|
||||||
|
return 1
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = args.output.with_suffix(args.output.suffix + ".tmp")
|
||||||
|
temporary.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
temporary.chmod(0o644)
|
||||||
|
temporary.replace(args.output)
|
||||||
|
print(f"Runtime image smoke evidence written to {args.output}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -6,11 +6,13 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
from hashlib import sha256
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import zipapp
|
from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent
|
ROOT = Path(__file__).resolve().parent
|
||||||
DEFAULT_OUTPUT = ROOT.parent.parent / "runtime" / "deployment" / "govoplan-deploy.pyz"
|
DEFAULT_OUTPUT = ROOT.parent.parent / "runtime" / "deployment" / "govoplan-deploy.pyz"
|
||||||
|
ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0)
|
||||||
|
PYTHON_FILE_MODE = 0o100644
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
@@ -25,13 +27,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
temporary = output.with_name(f".{output.name}.tmp")
|
temporary = output.with_name(f".{output.name}.tmp")
|
||||||
if temporary.exists():
|
if temporary.exists():
|
||||||
temporary.unlink()
|
temporary.unlink()
|
||||||
zipapp.create_archive(
|
_write_reproducible_zipapp(temporary)
|
||||||
ROOT,
|
|
||||||
target=temporary,
|
|
||||||
interpreter="/usr/bin/env python3",
|
|
||||||
compressed=True,
|
|
||||||
filter=_include_source,
|
|
||||||
)
|
|
||||||
temporary.chmod(0o755)
|
temporary.chmod(0o755)
|
||||||
temporary.replace(output)
|
temporary.replace(output)
|
||||||
digest = sha256(output.read_bytes()).hexdigest()
|
digest = sha256(output.read_bytes()).hexdigest()
|
||||||
@@ -39,6 +35,42 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _write_reproducible_zipapp(target: Path) -> None:
|
||||||
|
sources = tuple(
|
||||||
|
path
|
||||||
|
for path in sorted(ROOT.rglob("*"), key=lambda item: item.as_posix())
|
||||||
|
if path.is_file() and _include_source(path.relative_to(ROOT))
|
||||||
|
)
|
||||||
|
if not any(path.relative_to(ROOT).as_posix() == "__main__.py" for path in sources):
|
||||||
|
raise ValueError("deployment source has no __main__.py")
|
||||||
|
for path in sources:
|
||||||
|
if path.is_symlink():
|
||||||
|
raise ValueError(f"deployment source must not contain symlinks: {path}")
|
||||||
|
|
||||||
|
with target.open("wb") as handle:
|
||||||
|
handle.write(b"#!/usr/bin/env python3\n")
|
||||||
|
with ZipFile(
|
||||||
|
handle,
|
||||||
|
mode="w",
|
||||||
|
compression=ZIP_DEFLATED,
|
||||||
|
compresslevel=9,
|
||||||
|
strict_timestamps=True,
|
||||||
|
) as archive:
|
||||||
|
for source in sources:
|
||||||
|
relative = source.relative_to(ROOT).as_posix()
|
||||||
|
info = ZipInfo(relative, date_time=ZIP_TIMESTAMP)
|
||||||
|
info.compress_type = ZIP_DEFLATED
|
||||||
|
info.create_system = 3
|
||||||
|
info.external_attr = PYTHON_FILE_MODE << 16
|
||||||
|
info.flag_bits |= 0x800
|
||||||
|
archive.writestr(
|
||||||
|
info,
|
||||||
|
source.read_bytes(),
|
||||||
|
compress_type=ZIP_DEFLATED,
|
||||||
|
compresslevel=9,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _include_source(path: Path) -> bool:
|
def _include_source(path: Path) -> bool:
|
||||||
return (
|
return (
|
||||||
"__pycache__" not in path.parts
|
"__pycache__" not in path.parts
|
||||||
|
|||||||
@@ -627,6 +627,7 @@ def render_compose(spec: InstallationSpec) -> dict[str, object]:
|
|||||||
"tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"],
|
"tmpfs": ["/tmp:rw,noexec,nosuid,size=64m"],
|
||||||
"security_opt": ["no-new-privileges:true"],
|
"security_opt": ["no-new-privileges:true"],
|
||||||
"cap_drop": ["ALL"],
|
"cap_drop": ["ALL"],
|
||||||
|
"cap_add": ["NET_BIND_SERVICE"],
|
||||||
"volumes": [
|
"volumes": [
|
||||||
f"./{CADDY_CONFIG_FILENAME}:/etc/caddy/Caddyfile:ro",
|
f"./{CADDY_CONFIG_FILENAME}:/etc/caddy/Caddyfile:ro",
|
||||||
"caddy-data:/data",
|
"caddy-data:/data",
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import json
|
|||||||
import mimetypes
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import re
|
||||||
import secrets
|
import secrets
|
||||||
import sys
|
import sys
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -18,6 +19,7 @@ from urllib.request import Request, urlopen
|
|||||||
|
|
||||||
|
|
||||||
MAX_ASSET_BYTES = 256 * 1024 * 1024
|
MAX_ASSET_BYTES = 256 * 1024 * 1024
|
||||||
|
COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
|
||||||
|
|
||||||
class PublishError(RuntimeError):
|
class PublishError(RuntimeError):
|
||||||
@@ -30,6 +32,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
parser.add_argument("--owner", default="GovOPlaN")
|
parser.add_argument("--owner", default="GovOPlaN")
|
||||||
parser.add_argument("--repo", default="govoplan")
|
parser.add_argument("--repo", default="govoplan")
|
||||||
parser.add_argument("--tag", required=True)
|
parser.add_argument("--tag", required=True)
|
||||||
|
parser.add_argument("--target-commit", required=True)
|
||||||
parser.add_argument("--title", required=True)
|
parser.add_argument("--title", required=True)
|
||||||
parser.add_argument("--body", default="Signed GovOPlaN runtime distribution.")
|
parser.add_argument("--body", default="Signed GovOPlaN runtime distribution.")
|
||||||
parser.add_argument("--asset", type=Path, action="append", default=[], required=True)
|
parser.add_argument("--asset", type=Path, action="append", default=[], required=True)
|
||||||
@@ -48,25 +51,49 @@ class GiteaReleasePublisher:
|
|||||||
self.repo = repo
|
self.repo = repo
|
||||||
self.token = token
|
self.token = token
|
||||||
|
|
||||||
def release(self, *, tag: str, title: str, body: str) -> dict[str, Any]:
|
def release(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tag: str,
|
||||||
|
target_commit: str,
|
||||||
|
title: str,
|
||||||
|
body: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if COMMIT_SHA.fullmatch(target_commit) is None:
|
||||||
|
raise PublishError("release target must be an exact lowercase commit SHA")
|
||||||
|
resolved_target = self._resolve_commit(target_commit)
|
||||||
|
if resolved_target != target_commit:
|
||||||
|
raise PublishError("release target did not resolve to the requested commit")
|
||||||
|
existing_tag = self._resolve_commit(tag, allow_missing=True)
|
||||||
|
if existing_tag is not None and existing_tag != target_commit:
|
||||||
|
raise PublishError(
|
||||||
|
f"release tag {tag!r} already points to another commit"
|
||||||
|
)
|
||||||
|
|
||||||
path = self._repo_path(f"/releases/tags/{quote(tag, safe='')}")
|
path = self._repo_path(f"/releases/tags/{quote(tag, safe='')}")
|
||||||
try:
|
try:
|
||||||
return self._json("GET", path)
|
release = self._json("GET", path)
|
||||||
except HTTPError as exc:
|
except HTTPError as exc:
|
||||||
if exc.code != 404:
|
if exc.code != 404:
|
||||||
raise
|
raise
|
||||||
return self._json(
|
release = self._json(
|
||||||
"POST",
|
"POST",
|
||||||
self._repo_path("/releases"),
|
self._repo_path("/releases"),
|
||||||
payload={
|
payload={
|
||||||
"tag_name": tag,
|
"tag_name": tag,
|
||||||
"name": title,
|
"target_commitish": target_commit,
|
||||||
"body": body,
|
"name": title,
|
||||||
"draft": False,
|
"body": body,
|
||||||
"prerelease": False,
|
"draft": False,
|
||||||
},
|
"prerelease": False,
|
||||||
expected=201,
|
},
|
||||||
)
|
expected=201,
|
||||||
|
)
|
||||||
|
if self._resolve_commit(tag) != target_commit:
|
||||||
|
raise PublishError(
|
||||||
|
f"release tag {tag!r} does not resolve to the requested commit"
|
||||||
|
)
|
||||||
|
return release
|
||||||
|
|
||||||
def upload_assets(self, release: dict[str, Any], assets: tuple[Path, ...]) -> None:
|
def upload_assets(self, release: dict[str, Any], assets: tuple[Path, ...]) -> None:
|
||||||
release_id = release.get("id")
|
release_id = release.get("id")
|
||||||
@@ -165,6 +192,21 @@ class GiteaReleasePublisher:
|
|||||||
def _headers(self) -> dict[str, str]:
|
def _headers(self) -> dict[str, str]:
|
||||||
return {"Authorization": f"token {self.token}", "Accept": "application/json"}
|
return {"Authorization": f"token {self.token}", "Accept": "application/json"}
|
||||||
|
|
||||||
|
def _resolve_commit(self, ref: str, *, allow_missing: bool = False) -> str | None:
|
||||||
|
path = self._repo_path(f"/git/commits/{quote(ref, safe='')}")
|
||||||
|
try:
|
||||||
|
commit = self._json("GET", path)
|
||||||
|
except HTTPError as exc:
|
||||||
|
if allow_missing and exc.code == 404:
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
if not isinstance(commit, dict):
|
||||||
|
raise PublishError(f"Gitea returned an invalid commit for {ref!r}")
|
||||||
|
sha = commit.get("sha")
|
||||||
|
if not isinstance(sha, str) or COMMIT_SHA.fullmatch(sha) is None:
|
||||||
|
raise PublishError(f"Gitea returned an invalid commit SHA for {ref!r}")
|
||||||
|
return sha
|
||||||
|
|
||||||
def _repo_path(self, suffix: str) -> str:
|
def _repo_path(self, suffix: str) -> str:
|
||||||
return (
|
return (
|
||||||
f"{self.base_url}/api/v1/repos/{quote(self.owner, safe='')}/"
|
f"{self.base_url}/api/v1/repos/{quote(self.owner, safe='')}/"
|
||||||
@@ -189,7 +231,12 @@ def main() -> int:
|
|||||||
repo=args.repo,
|
repo=args.repo,
|
||||||
token=os.environ.get(args.token_env, ""),
|
token=os.environ.get(args.token_env, ""),
|
||||||
)
|
)
|
||||||
release = publisher.release(tag=args.tag, title=args.title, body=args.body)
|
release = publisher.release(
|
||||||
|
tag=args.tag,
|
||||||
|
target_commit=args.target_commit,
|
||||||
|
title=args.title,
|
||||||
|
body=args.body,
|
||||||
|
)
|
||||||
publisher.upload_assets(release, tuple(args.asset))
|
publisher.upload_assets(release, tuple(args.asset))
|
||||||
except (HTTPError, OSError, PublishError, ValueError) as exc:
|
except (HTTPError, OSError, PublishError, ValueError) as exc:
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
|||||||
@@ -21,6 +21,12 @@ def resolve_platforms(
|
|||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
if not repository or "@" in repository or any(value.isspace() for value in repository):
|
if not repository or "@" in repository or any(value.isspace() for value in repository):
|
||||||
raise ValueError("repository must be an unpinned OCI repository name")
|
raise ValueError("repository must be an unpinned OCI repository name")
|
||||||
|
last_slash = repository.rfind("/")
|
||||||
|
last_colon = repository.rfind(":")
|
||||||
|
if last_colon > last_slash:
|
||||||
|
repository = repository[:last_colon]
|
||||||
|
if not repository:
|
||||||
|
raise ValueError("repository must be an unpinned OCI repository name")
|
||||||
if DIGEST.fullmatch(index_digest) is None:
|
if DIGEST.fullmatch(index_digest) is None:
|
||||||
raise ValueError("index digest must be sha256:<hex>")
|
raise ValueError("index digest must be sha256:<hex>")
|
||||||
if not isinstance(payload, dict) or not isinstance(payload.get("manifests"), list):
|
if not isinstance(payload, dict) or not isinstance(payload.get("manifests"), list):
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ ENV PYTHONUNBUFFERED=1 \
|
|||||||
PYTHONDONTWRITEBYTECODE=1 \
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
PYTHONPATH=/opt/govoplan/runtime \
|
PYTHONPATH=/opt/govoplan/runtime \
|
||||||
PATH=/opt/govoplan/runtime/bin:${PATH} \
|
PATH=/opt/govoplan/runtime/bin:${PATH} \
|
||||||
|
GOVOPLAN_CORE_SOURCE_ROOT=/opt/govoplan/runtime/govoplan_core_runtime \
|
||||||
HOME=/var/lib/govoplan
|
HOME=/var/lib/govoplan
|
||||||
|
|
||||||
COPY wheelhouse/ /opt/govoplan/wheels/
|
COPY wheelhouse/ /opt/govoplan/wheels/
|
||||||
|
|||||||
@@ -13,7 +13,10 @@ http {
|
|||||||
sendfile on;
|
sendfile on;
|
||||||
server_tokens off;
|
server_tokens off;
|
||||||
client_body_temp_path /tmp/client_temp;
|
client_body_temp_path /tmp/client_temp;
|
||||||
|
fastcgi_temp_path /tmp/fastcgi_temp;
|
||||||
proxy_temp_path /tmp/proxy_temp;
|
proxy_temp_path /tmp/proxy_temp;
|
||||||
|
scgi_temp_path /tmp/scgi_temp;
|
||||||
|
uwsgi_temp_path /tmp/uwsgi_temp;
|
||||||
|
|
||||||
map $http_x_forwarded_proto $govoplan_forwarded_proto {
|
map $http_x_forwarded_proto $govoplan_forwarded_proto {
|
||||||
default $scheme;
|
default $scheme;
|
||||||
|
|||||||
@@ -149,6 +149,7 @@ def main() -> int:
|
|||||||
requirements=requirements,
|
requirements=requirements,
|
||||||
python=python,
|
python=python,
|
||||||
local_requirements=local_requirements,
|
local_requirements=local_requirements,
|
||||||
|
repair_requirements=environment.stale_requirements,
|
||||||
force=args.force,
|
force=args.force,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -250,6 +251,7 @@ def build_install_plan(
|
|||||||
python: str,
|
python: str,
|
||||||
local_requirements: tuple[RequirementEntry, ...],
|
local_requirements: tuple[RequirementEntry, ...],
|
||||||
force: bool,
|
force: bool,
|
||||||
|
repair_requirements: tuple[RequirementEntry, ...] = (),
|
||||||
) -> InstallPlan:
|
) -> InstallPlan:
|
||||||
full_command = (python, "-m", "pip", "install", "-r", str(requirements))
|
full_command = (python, "-m", "pip", "install", "-r", str(requirements))
|
||||||
if force:
|
if force:
|
||||||
@@ -273,7 +275,12 @@ def build_install_plan(
|
|||||||
removed_paths = set(previous_inputs) - set(current_inputs)
|
removed_paths = set(previous_inputs) - set(current_inputs)
|
||||||
stale_requirements = requirements_key in changed_paths or requirements_key in removed_paths
|
stale_requirements = requirements_key in changed_paths or requirements_key in removed_paths
|
||||||
|
|
||||||
installs: list[tuple[str, ...]] = []
|
# Metadata changes and installation drift can happen together. Keep both
|
||||||
|
# sets in one resolver transaction so a selective metadata sync also
|
||||||
|
# repairs local distributions omitted from the current environment.
|
||||||
|
installs: list[tuple[str, ...]] = [
|
||||||
|
requirement.install_args for requirement in repair_requirements
|
||||||
|
]
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
|
|
||||||
if stale_requirements:
|
if stale_requirements:
|
||||||
@@ -322,7 +329,7 @@ def build_install_plan(
|
|||||||
)
|
)
|
||||||
return InstallPlan(
|
return InstallPlan(
|
||||||
"Selective Python environment sync",
|
"Selective Python environment sync",
|
||||||
f"installing {len(deduped_installs)} stale local requirement(s) in one resolver transaction.",
|
f"installing {len(deduped_installs)} stale or missing local requirement(s) in one resolver transaction.",
|
||||||
(command,),
|
(command,),
|
||||||
tuple(warnings),
|
tuple(warnings),
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user