add-ideas/govoplan
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Initialize GovOPlaN meta repository

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-11 17:16:58 +02:00
commit 70109afc78
70 changed files with 11641 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.dockerignore Normal file
View File

@@ -0,0 +1,28 @@
.git
.venv
node_modules
webui/node_modules
audit-reports
runtime
dist
build
coverage
htmlcov
__pycache__
*.pyc
*.pyo
*.log
.mypy_cache
.pytest_cache
.ruff_cache
.cache
.component-test-build
.module-test-build
.policy-test-build
.template-preview-test-build
.import-test-build
webui/.component-test-build
webui/.module-test-build
webui/.policy-test-build
webui/.template-preview-test-build
webui/.import-test-build

32
.env.example Normal file
View File

@@ -0,0 +1,32 @@
# GovOPlaN self-hosted install configuration.
# Copy to a deployment-local .env or secret store. Do not commit populated secrets.
APP_ENV=production
GOVOPLAN_INSTALL_PROFILE=self-hosted
MASTER_KEY_B64=<generate-with-govoplan-config-env-template-generate-secrets>
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,docs,ops
CELERY_ENABLED=true
REDIS_URL=redis://127.0.0.1:6379/0
CELERY_QUEUES=send_email,append_sent,default
CORS_ORIGINS=https://govoplan.example.org
AUTH_COOKIE_SECURE=true
AUTH_COOKIE_SAMESITE=lax
AUTH_COOKIE_DOMAIN=
FILE_STORAGE_BACKEND=local
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
DEV_AUTO_MIGRATE_ENABLED=false
DEV_BOOTSTRAP_ENABLED=false
DEV_MAILBOX_API_ENABLED=false
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable

35
.gitea/ISSUE_TEMPLATE/bug_report.md Normal file
View File

@@ -0,0 +1,35 @@
---
name: "Bug"
about: "Report a reproducible defect, regression, or incorrect behavior"
title: "[Bug] "
labels:
- type/bug
- status/triage
- module/core
---
## Scope
- Repository:
- Area/module:
- Affected version or commit:
## Behavior
Expected:
Actual:
## Reproduction
1.
2.
3.
## Evidence
Logs, screenshots, traces, or failing test output:
## Verification Target
Command or workflow that should pass when fixed:

1
.gitea/ISSUE_TEMPLATE/config.yaml Normal file
View File

@@ -0,0 +1 @@
blank_issues_enabled: false

27
.gitea/ISSUE_TEMPLATE/docs_workflow.md Normal file
View File

@@ -0,0 +1,27 @@
---
name: "Docs / workflow"
about: "Request documentation, process, or developer workflow changes"
title: "[Docs] "
labels:
- type/docs
- status/triage
- module/core
- area/docs
---
## Scope
- Repository:
- Document or workflow:
## Current State
What is missing, unclear, duplicated, or stale?
## Desired State
What should the docs or workflow make clear?
## Verification Target
How should this be checked?

32
.gitea/ISSUE_TEMPLATE/feature_request.md Normal file
View File

@@ -0,0 +1,32 @@
---
name: "Feature"
about: "Propose new user-visible behavior or platform capability"
title: "[Feature] "
labels:
- type/feature
- status/triage
- module/core
---
## Problem
What user, operator, or developer problem should this solve?
## Proposed Capability
What should exist when this is done?
## Ownership
- Owning repository:
- Related module repositories:
- Extension point or integration boundary:
## Acceptance Criteria
- [ ]
- [ ]
## Verification Target
Command, scenario, or UI flow that should prove completion:

28
.gitea/ISSUE_TEMPLATE/task.md Normal file
View File

@@ -0,0 +1,28 @@
---
name: "Task"
about: "Track implementation, maintenance, or migration work"
title: "[Task] "
labels:
- type/task
- status/triage
- module/core
---
## Objective
What needs to be completed?
## Scope
- Owning repository:
- In-scope:
- Out-of-scope:
## Checklist
- [ ]
- [ ]
## Verification Target
Command or manual check:

25
.gitea/ISSUE_TEMPLATE/tech_debt.md Normal file
View File

@@ -0,0 +1,25 @@
---
name: "Tech debt"
about: "Track cleanup, refactoring, risk reduction, or deferred engineering work"
title: "[Debt] "
labels:
- type/debt
- status/triage
- module/core
---
## Current Cost
What does this make harder, riskier, slower, or more fragile?
## Desired Shape
What should the code, tests, or architecture look like afterwards?
## Constraints
What behavior, compatibility, or module boundary must be preserved?
## Verification Target
Focused checks that should pass:

15
.gitea/PULL_REQUEST_TEMPLATE.md Normal file
View File

@@ -0,0 +1,15 @@
## Issue
Closes #
## Summary
-
## Verification
-
## Notes
Follow-up issues:

53
.gitea/workflows/dependency-audit.yml Normal file
View File

@@ -0,0 +1,53 @@
name: Dependency Audit
on:
pull_request:
push:
branches:
- main
schedule:
- cron: "23 3 * * 1"
jobs:
dependency-audit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
path: govoplan
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Configure SSH for release dependencies
env:
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
run: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
exit 1
fi
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
- name: Bootstrap GovOPlaN repositories
working-directory: govoplan
run: python tools/repo/bootstrap-repositories.py --parent ..
- name: Install backend dev audit dependencies
working-directory: govoplan
run: |
python -m venv .venv
.venv/bin/python -m pip install --upgrade pip
.venv/bin/python -m pip install -r requirements-release.txt
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
- name: Install WebUI release dependencies
working-directory: govoplan
run: bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
- name: Run dependency audits
working-directory: govoplan
run: GOVOPLAN_CORE_ROOT="$PWD/../govoplan-core" PYTHON="$PWD/.venv/bin/python" bash tools/checks/check-dependency-audits.sh

49
.gitea/workflows/module-matrix.yml Normal file
View File

@@ -0,0 +1,49 @@
name: Module Matrix
on:
workflow_dispatch:
pull_request:
jobs:
module-matrix:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
path: govoplan
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Configure SSH for release dependencies
env:
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
run: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
exit 1
fi
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
- name: Bootstrap GovOPlaN repositories
working-directory: govoplan
run: python tools/repo/bootstrap-repositories.py --parent ..
- name: Install backend release dependencies
working-directory: govoplan
run: |
python -m venv .venv
.venv/bin/python -m pip install --upgrade pip
.venv/bin/python -m pip install -r requirements-release.txt
.venv/bin/python -m pip install '../govoplan-core[dev]'
- name: Install WebUI release dependencies with test scripts
working-directory: govoplan
run: bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
- name: Run module matrix and contract tests
working-directory: govoplan
run: GOVOPLAN_CORE_ROOT="$PWD/../govoplan-core" PYTHON="$PWD/.venv/bin/python" bash tools/checks/check-module-matrix.sh

48
.gitea/workflows/release-integration.yml Normal file
View File

@@ -0,0 +1,48 @@
name: Release Integration
on:
workflow_dispatch:
jobs:
release-integration:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
path: govoplan
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: "3.12"
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
with:
node-version: "22"
- name: Configure SSH for release dependencies
env:
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
run: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
exit 1
fi
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
- name: Bootstrap GovOPlaN repositories
working-directory: govoplan
run: python tools/repo/bootstrap-repositories.py --parent ..
- name: Install backend release integration dependencies
working-directory: govoplan
run: |
python -m venv .venv
.venv/bin/python -m pip install --upgrade pip
.venv/bin/python -m pip install -r requirements-release.txt
.venv/bin/python -m pip install '../govoplan-core[dev]'
- name: Install WebUI release dependencies
working-directory: govoplan
run: bash tools/release/install-webui-release-dependencies.sh ../govoplan-core/webui
- name: Run release integration checks
working-directory: govoplan
run: GOVOPLAN_CORE_ROOT="$PWD/../govoplan-core" PYTHON="$PWD/.venv/bin/python" bash tools/checks/check-release-integration.sh

20
.gitea/workflows/security-audit-toolbox-update.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Security Audit Toolbox Update
on:
schedule:
- cron: "17 1 * * 6"
workflow_dispatch:
jobs:
toolbox-update:
runs-on: ubuntu-latest
env:
SECURITY_AUDIT_UPDATE: "1"
SECURITY_AUDIT_REQUIRE_TOOLS: "1"
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
path: govoplan
- name: Refresh audit toolbox image
working-directory: govoplan
run: tools/checks/security-audit/run.sh --mode quick --scope current --build-only

49
.gitea/workflows/security-audit.yml Normal file
View File

@@ -0,0 +1,49 @@
name: Security Audit
on:
pull_request:
push:
branches:
- main
schedule:
- cron: "41 2 * * 2"
workflow_dispatch:
jobs:
security-audit:
runs-on: ubuntu-latest
env:
SECURITY_AUDIT_MODE: ci
SECURITY_AUDIT_SCOPE: govoplan
SECURITY_AUDIT_FAIL_ON_FINDINGS: "0"
SECURITY_AUDIT_REQUIRE_TOOLS: "1"
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
path: govoplan
- name: Configure SSH for repository bootstrap
env:
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
run: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh repository bootstrap."
exit 1
fi
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
- name: Bootstrap GovOPlaN repositories
working-directory: govoplan
run: python tools/repo/bootstrap-repositories.py --parent ..
- name: Run security audit
working-directory: govoplan
run: tools/checks/security-audit/run.sh --mode "$SECURITY_AUDIT_MODE" --scope "$SECURITY_AUDIT_SCOPE" --reports-dir audit-reports
- name: Upload audit reports
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
with:
name: security-audit-reports
path: govoplan/audit-reports

10
.gitignore vendored Normal file
View File

@@ -0,0 +1,10 @@
.cache/
.mypy_cache/
.pytest_cache/
.ruff_cache/
.venv/
runtime/
__pycache__/
audit-reports/
coverage/
htmlcov/

18
.gitleaks.toml Normal file
View File

@@ -0,0 +1,18 @@
title = "GovOPlaN secret scanning"
[allowlist]
description = "Repository examples and documented development placeholders"
regexes = [
'''dev-multimailer-api-key''',
'''example(\.invalid|\.org|\.com)''',
'''postgresql(\+psycopg)?://[^:@/\s]+:\*\*\*@''',
'''<redacted>''',
]
paths = [
'''(^|/)audit-reports/''',
'''(^|/)node_modules/''',
'''(^|/)\.venv/''',
'''(^|/)runtime/''',
'''(^|/)dist/''',
'''(^|/)build/''',
]

115
README.md Normal file
View File

@@ -0,0 +1,115 @@
# GovOPlaN
<!-- govoplan-repository-type:start -->
**Repository type:** system (meta).
<!-- govoplan-repository-type:end -->
This is the GovOPlaN meta repository. It is the operator entry point for
whole-product development, release orchestration, repository bootstrap, and
system-level Docker composition.
It is not a runtime module. Runtime behavior belongs to `govoplan-core` and the
installed modules/connectors discovered by core.
## Common Commands
Create the whole-product development virtualenv in this meta repository:
```sh
python3 -m venv .venv
./.venv/bin/python -m pip install --upgrade pip
./.venv/bin/python -m pip install -r requirements-dev.txt
```
The meta venv is the default Python environment for launch, check, release, and
Gitea tooling. `GOVOPLAN_VENV_ROOT` or `PYTHON` can override it for special
cases.
Start the development stack through the meta repository:
```sh
./tools/launch/launch-dev.sh
```
Start the shared development PostgreSQL service:
```sh
./tools/launch/start-dev-postgres.sh
```
Check which GovOPlaN repositories are present and dirty:
```sh
./tools/repo/repo-status.sh
```
Clone missing repositories listed in `repositories.json`:
```sh
./tools/repo/bootstrap-repositories.py
```
Update generated repository type notes in all READMEs:
```sh
./tools/repo/update-repository-type-notes.py
```
Run the consolidated focused verification suite:
```sh
./tools/checks/check-focused.sh
```
Run the cross-repository dependency boundary gate:
```sh
./tools/checks/check_dependency_boundaries.py
```
Run installer rollback drills:
```sh
./tools/checks/module-installer-rollback-drill.py --format json
```
Release, catalog, Gitea, security-audit, and cross-repository maintenance
commands should also be called from this repository through `tools/`.
## Configuration
The repository root `.env.example` is the self-hosted operator template for a
full GovOPlaN installation. Development profile examples live below `dev/`, for
example `dev/postgres/.env.example` and `dev/production-like/.env.example`.
Do not commit populated `.env` files. Gitea tokens should stay in a local file
such as `~/.config/gitea/gitea.env` and be passed with `--env-file`.
## Structure
The repository categories are documented in
`docs/REPOSITORY_STRUCTURE.md`. The machine-readable list lives in
`repositories.json`.
# GovOPlaN Docker
Whole-product Docker composition belongs in this meta repository.
Current module-specific Docker test beds remain in their owning repositories
until they are migrated or wrapped here:
- `govoplan/dev/postgres`
- `govoplan/dev/production-like`
- `govoplan-campaign/dev/mail-testbed`
- `govoplan-files/dev/connectors`
The target shape is:
- `govoplan/dev/postgres`: shared local development PostgreSQL service.
- `govoplan/dev/production-like`: production-like validation composition.
- module repositories keep only narrow connector or protocol test beds.
What doesn't belong here:
- `addideas-govoplan-website/docker-compose.yml`: public website serving
profile; this one stays with the website repository.

5
dev/postgres/.env.example Normal file
View File

@@ -0,0 +1,5 @@
GOVOPLAN_POSTGRES_DB=govoplan
GOVOPLAN_POSTGRES_USER=govoplan
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
GOVOPLAN_POSTGRES_PORT=55432
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan

127
dev/postgres/README.md Normal file
View File

@@ -0,0 +1,127 @@
# PostgreSQL Development Profile
GovOPlaN development now defaults to PostgreSQL:
```text
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
```
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
```text
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
```
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
The devserver and `govoplan/tools/launch/launch-dev.sh` honor an explicit
`DATABASE_URL` if you need a different database.
## Host PostgreSQL Setup
Create the default local role and database from a host shell:
```bash
export GOVOPLAN_PG_DB=govoplan_dev
export GOVOPLAN_PG_USER=govoplan_dev
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
echo
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
else
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
fi
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
fi
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
touch ~/.pgpass
chmod 600 ~/.pgpass
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
mv ~/.pgpass.tmp ~/.pgpass
chmod 600 ~/.pgpass
```
Check access:
```bash
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
-c 'select current_user, current_database();'
```
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
available through:
```bash
flatpak-spawn --host /usr/bin/psql --version
flatpak-spawn --host /usr/bin/pg_dump --version
flatpak-spawn --host /usr/bin/pg_restore --version
```
## Run GovOPlaN Against Host PostgreSQL
```bash
cd /mnt/DATA/git/govoplan
./.venv/bin/python -m govoplan_core.commands.init_db \
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
--with-dev-data
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
```
For the full backend and WebUI launcher:
```bash
cd /mnt/DATA/git/govoplan
tools/launch/launch-dev.sh
```
To force the old SQLite fallback for a disposable local run:
```bash
cd /mnt/DATA/git/govoplan
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite tools/launch/launch-dev.sh
```
## Disposable Docker Testbed
This testbed is for migration and module-permutation checks. It is not a
production deployment profile.
```bash
cd /mnt/DATA/git/govoplan/dev/postgres
cp .env.example .env
docker compose --env-file .env up -d
```
Run the integration check from the meta checkout:
```bash
cd /mnt/DATA/git/govoplan
set -a
. dev/postgres/.env
set +a
tools/checks/postgres-integration-check.py \
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
--reset-schema
```
`--reset-schema` drops and recreates the `public` schema before every module
set. Use it only against this disposable database.
Stop the testbed:
```bash
cd /mnt/DATA/git/govoplan/dev/postgres
docker compose --env-file .env down
```
Remove all test data:
```bash
docker compose --env-file .env down -v
```

22
dev/postgres/docker-compose.yml Normal file
View File

@@ -0,0 +1,22 @@
services:
postgres:
image: postgres:16-alpine
container_name: govoplan-postgres-dev
environment:
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
ports:
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
healthcheck:
test:
- CMD-SHELL
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
interval: 5s
timeout: 3s
retries: 20
volumes:
- postgres-data:/var/lib/postgresql/data
volumes:
postgres-data:

27
dev/production-like/.env.example Normal file
View File

@@ -0,0 +1,27 @@
APP_ENV=staging
GOVOPLAN_INSTALL_PROFILE=production-like
MASTER_KEY_B64=
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
REDIS_URL=redis://127.0.0.1:56379/0
CELERY_ENABLED=true
CELERY_QUEUES=send_email,append_sent,default
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
AUTH_COOKIE_SECURE=false
FILE_STORAGE_BACKEND=local
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
DEV_AUTO_MIGRATE_ENABLED=false
DEV_BOOTSTRAP_ENABLED=true

60
dev/production-like/README.md Normal file
View File

@@ -0,0 +1,60 @@
# Production-Like Development Profile
This profile runs the shared services that production depends on while keeping
API, worker, and WebUI code in the editable local repositories.
It provides:
- PostgreSQL with a persistent Docker volume
- Redis with append-only persistence
- explicit `ENABLED_MODULES`
- local durable file storage under `runtime/production-like/files`
- a Celery worker process using the same queues as the API
Start it from the meta repository:
```bash
cd /mnt/DATA/git/govoplan
tools/launch/launch-production-like-dev.sh
```
The helper wrapper exposes repeatable lifecycle commands:
```bash
cd /mnt/DATA/git/govoplan
tools/launch/production-like-dev.sh validate-config
tools/launch/production-like-dev.sh seed
tools/launch/production-like-dev.sh start
tools/launch/production-like-dev.sh stop
tools/launch/production-like-dev.sh reset --yes
```
The launcher uses `dev/production-like/.env` when present, otherwise
`dev/production-like/.env.example`. Copy the example when you want local port or
password changes:
```bash
cp dev/production-like/.env.example dev/production-like/.env
```
The API and worker use:
```text
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
REDIS_URL=redis://127.0.0.1:56379/0
CELERY_ENABLED=true
```
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
containers keep running by default so the next launch is fast. To stop them too:
```bash
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 tools/launch/launch-production-like-dev.sh
```
To remove all profile data:
```bash
cd /mnt/DATA/git/govoplan
tools/launch/production-like-dev.sh reset --yes
```

37
dev/production-like/docker-compose.yml Normal file
View File

@@ -0,0 +1,37 @@
services:
postgres:
image: postgres:16-alpine
container_name: govoplan-production-like-postgres
environment:
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
ports:
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
healthcheck:
test:
- CMD-SHELL
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
interval: 5s
timeout: 3s
retries: 20
volumes:
- postgres-data:/var/lib/postgresql/data
redis:
image: redis:7-alpine
container_name: govoplan-production-like-redis
command: ["redis-server", "--appendonly", "yes"]
ports:
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
volumes:
- redis-data:/data
volumes:
postgres-data:
redis-data:

17
docker/README.md Normal file
View File

@@ -0,0 +1,17 @@
# GovOPlaN Docker
Whole-product Docker composition belongs in this meta repository.
Current shared profiles:
- `govoplan/dev/postgres`
- `govoplan/dev/production-like`
Module-specific Docker test beds remain in their owning repositories:
- `govoplan-campaign/dev/mail-testbed`
- `govoplan-files/dev/connectors`
Website-specific serving profiles remain with the public website repository:
- `addideas-govoplan-website/docker-compose.yml`

321
docs/GITEA_ISSUES.md Normal file
View File

@@ -0,0 +1,321 @@
# Gitea Issues And Wiki Workflow
Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes.
The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state.
## Initial Setup
The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`.
The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan.git`). Override inference when needed:
```bash
export GITEA_URL=https://git.add-ideas.de
export GITEA_OWNER=add-ideas
export GITEA_REPO=govoplan
export GITEA_TOKEN=...
```
The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens:
```bash
GITEA_TOKEN=...
# Optional if origin inference is not enough:
GITEA_URL=https://git.add-ideas.de
GITEA_OWNER=add-ideas
GITEA_REPO=govoplan
```
For a shared credentials file outside the target repository, pass `--env-file`:
```bash
./tools/gitea/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
```
Create a Gitea token with issue read/write access and label-management
permission for the repository. On scoped-token instances, this usually means
issue read/write and, if label writes are rejected, repository write permission
too.
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
Creating or updating organization labels requires a token with
`write:organization`. Repository label management only needs repository label
permission, but it duplicates the taxonomy into each repository.
Preview and apply labels:
```bash
cd /mnt/DATA/git/govoplan
./tools/gitea/gitea-sync-labels.py
./tools/gitea/gitea-sync-labels.py --apply
```
Preview and apply the shared taxonomy as organization labels:
```bash
cd /mnt/DATA/git/govoplan
./tools/gitea/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
./tools/gitea/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
```
The import helpers resolve repository labels and organization labels. Repository
labels win when a repository defines the same name locally, but a repository does
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
`module/*`, `area/*`, `source/*`, or `codex/*` label.
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
## Multiple Repositories And Workspaces
The helper scripts are path-based and can run from the meta checkout against any repository with a Gitea remote:
```bash
cd /mnt/DATA/git/govoplan
./tools/gitea/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply
./tools/gitea/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail
./tools/gitea/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress
```
Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH.
When the meta checkout drives another workspace, prefer `--env-file` for shared
credentials instead of putting `GITEA_REPO` in the meta `.env`; a repo-specific
`GITEA_REPO` can accidentally override target inference.
For non-GovOPlaN projects, provide a project-specific label file and module/project label:
```bash
./tools/gitea/gitea-sync-labels.py \
--root /path/to/project \
--labels-file /path/to/project/docs/gitea-labels.json \
--apply
./tools/gitea/gitea-todo-import.py \
--root /path/to/project \
--module-label project/example \
--extra-label area/backend
```
If another project does not use `area/*` labels, disable area inference:
```bash
./tools/gitea/gitea-todo-import.py \
--root /path/to/project \
--module-label project/example \
--no-area-labels
```
Install or refresh the shared issue templates in sibling or external repositories:
```bash
./tools/gitea/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail
./tools/gitea/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply
```
The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-<name>` checkout maps to `module/<name>`. For another workspace or repository name, pass an explicit label:
```bash
./tools/gitea/gitea-install-workflow.py /path/to/repo --module-label module/example --apply
```
Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in the meta repo and run the sync script from the meta repo.
For a fully portable workflow kit, copy these files into the other project:
- `tools/gitea/gitea_common.py`
- `tools/gitea/gitea-sync-labels.py`
- `tools/gitea/gitea-todo-import.py`
- `tools/gitea/gitea-codex-note.py`
- `tools/gitea/gitea-install-workflow.py`
- `.gitea/ISSUE_TEMPLATE/*`
- `.gitea/PULL_REQUEST_TEMPLATE.md`
- a project-specific `docs/gitea-labels.json`
Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup.
## Label Taxonomy
Use one `type/*` label:
- `type/bug`
- `type/feature`
- `type/task`
- `type/debt`
- `type/docs`
Use one `status/*` label while the issue is open:
- `status/triage`: needs ownership, priority, or acceptance criteria.
- `status/ready`: ready to implement.
- `status/in-progress`: actively being worked.
- `status/blocked`: blocked on an external dependency, credential, or decision.
- `status/needs-info`: blocked on clarification.
Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`.
Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository.
Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first.
## Moving TODOs Into Gitea
Preview inline markers:
```bash
cd /mnt/DATA/git/govoplan
./tools/gitea/gitea-todo-import.py
```
Create missing issues after labels are synced:
```bash
./tools/gitea/gitea-todo-import.py --apply
```
The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items.
When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference:
```python
# TODO(gitea#123): keep only if the local pointer is still useful
```
Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers.
For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer:
```bash
./tools/gitea/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env
./tools/gitea/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
```
It scans repository and product-directory files with backlog-like names, resolves
shared labels from the organization label catalogue where available, creates only
missing fallback repository labels when needed, imports missing open work, and
deduplicates reruns by hidden fingerprint and normalized title.
## Mirroring Project Docs Into Gitea Wikis
Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`:
```bash
cd /mnt/DATA/git/govoplan
./tools/gitea/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env
```
Apply the wiki mirror:
```bash
./tools/gitea/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
```
After renaming or deleting repository docs, prune previously managed wiki pages
that no longer have a source file:
```bash
./tools/gitea/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
```
The default apply path uses the Gitea wiki git repository, not one REST API
request per page. It keeps a local checkout cache below
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
pushes that commit. This is much faster and avoids REST wiki-page timeouts.
Use `--transport api` only when the wiki git remote is unavailable.
Limit a sync to one repository or one generated page while working:
```bash
./tools/gitea/gitea-sync-wiki.py --repo govoplan-core --apply
./tools/gitea/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply
```
Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository
sync when the set of mirrored pages changes.
The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related.
Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages.
Use the wiki for durable context:
- project overviews and architecture
- workflows, operating notes, and setup references
- product concepts, plans, pitches, and whitepapers
- historical context that helps interpret issues
Keep active state in issues:
- open tasks, TODOs, feature requests, and bugs
- priority, blocking status, and acceptance criteria
- Codex progress updates and implementation notes
## Codex State Updates
Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes:
- scope understood
- files changed
- tests or manual checks run
- blockers or decisions needed
- follow-up issues created
Preview and post a standardized note:
```bash
./tools/gitea/gitea-codex-note.py \
--issue 123 \
--status progress \
--summary "Implemented capability metadata fallback." \
--changed src/govoplan_core/modules/registry.py \
--test "./.venv/bin/python -m unittest tests.test_module_system"
./tools/gitea/gitea-codex-note.py \
--issue 123 \
--status progress \
--summary "Implemented capability metadata fallback." \
--changed src/govoplan_core/modules/registry.py \
--test "./.venv/bin/python -m unittest tests.test_module_system" \
--apply
```
Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded.
## Ownership Rules
Create the issue in the repository that owns the change:
- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components.
- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration.
- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations.
- `govoplan-files`: files-specific backend, frontend, storage, and file workflows.
- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior.
For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details.
## Cleaning Up Mirrored Sources
After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state.
Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding.
Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path.
## Docs Versus Issues
Keep durable facts in docs:
- architecture and extension points
- command references
- module boundaries
- operational conventions
Keep changing state in Gitea:
- TODOs and follow-ups
- bugs and feature requests
- blocked status
- acceptance criteria
- implementation notes from active work
If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history.

180
docs/META_REPOSITORY_MIGRATION_AUDIT.md Normal file
View File

@@ -0,0 +1,180 @@
# Meta Repository Migration Audit
This audit records which existing GovOPlaN files should move toward the
`govoplan` meta repository and which should remain with their current runtime
owner.
## Website Repository Rename
The former `govoplan-web` repository has been renamed to
`addideas-govoplan-website`. It is a public website and catalog publication
target, not an installable GovOPlaN runtime module.
Resolved target name:
- `addideas-govoplan-website`
The rename happened server-side in Gitea first. Local checkout paths, remotes,
docs, and release/catalog defaults should use the new name.
The renamed repository should remain categorized as `website`, not `module`.
## Move To `govoplan`
These are whole-product/operator concerns and should be owned by the meta
repository.
### Release Orchestration
Former location:
- core-local release and catalog helper scripts
Target shape:
- command entry points live in `govoplan/tools`
- repository lists come from `govoplan/repositories.json`
- generated catalogs are written to the renamed public site repository
- core keeps only runtime catalog validation/signing primitives used by the
installer
### Cross-Repository Gitea Workflow Tooling
Former location:
- core-local Gitea helper scripts
- source issue templates copied into each repo under `.gitea/`
- the shared label taxonomy JSON file
Target shape:
- source templates and label taxonomy live in `govoplan`
- sync/import/migration helpers live in `govoplan/tools`
- generated/copy-installed `.gitea/` files may remain in each repository
because Gitea reads them repository-locally
- repo-specific AGENTS or issue guidance can remain repo-local when it contains
module-specific instructions
### Whole-Product CI
Former location:
- core-local whole-product workflow definitions
Target shape:
- whole-product workflows live in `govoplan/.gitea/workflows`
- workflows bootstrap or use sibling checkouts based on `repositories.json`
- core keeps core-only workflows if needed later
Notes:
- Gitea workflows are repository-local, so moving them changes which repository
triggers the job.
- The workflow scripts must stop assuming the checked-out repository is
`govoplan-core`.
### Security Audit Toolbox
Former location:
- core-local security audit toolbox files
- core-local audit dependency and Gitleaks configuration files
Reason to move:
- the tool already supports `--scope govoplan`
- the `govoplan` scan mounts the parent directory and scans sibling repos
- this is an operator/security workspace function, not runtime core behavior
Target shape:
- `govoplan/tools/checks/security-audit/`
- `govoplan/tools/checks/check-security-audit.sh`
- `govoplan/requirements-audit.txt`
- `govoplan/.gitleaks.toml`
### Whole-Product Docker
Former locations:
- core-local shared development Docker profiles
- website-local public-site Compose profile
Target shape:
- shared development and production-like composition move under `govoplan/dev`
- deploy/public-site composition can live under `govoplan/docker`
- module-specific testbeds stay module-local:
- `govoplan-campaign/dev/mail-testbed/docker-compose.yml`
- `govoplan-files/dev/connectors/docker-compose.yml`
## Keep In Core
These are runtime/kernel concerns and should stay in `govoplan-core`.
- installer/runtime code under `govoplan_core.core.module_installer`
- module discovery, registry, route aggregation, and lifecycle contracts
- catalog validation primitives used by the runtime installer
- configuration package validation and safety contracts
- database/session primitives and migration orchestration APIs
- core WebUI shell and module contribution contracts
Canonical operator entry points should be in `govoplan`.
## Keep In Public Site Repository
After the rename, the website repository should keep:
- Vite/React public website source
- screenshots/content source notes
- static catalog assets under `public/catalogs/...`
- website-specific catalog asset validator if it validates static publication
contents
- website Dockerfile and simple site-serving compose, unless a whole-product
deployment compose takes over in `govoplan`
It should not own:
- release version bumping
- cross-repository tagging
- catalog generation/signing keys
- installer logic
- module orchestration
## Rename Follow-Up References
Known references reviewed after the server-side rename:
- `govoplan/repositories.json`
- `govoplan/docs/REPOSITORY_STRUCTURE.md`
- `govoplan/docker/README.md`
- `govoplan-core/docs/RELEASE_DEPENDENCIES.md`
- `govoplan-core/docs/MODULE_ARCHITECTURE.md`
- moved release helper references
- moved Gitea helper references
- public site `README.md`, `package.json`, `package-lock.json`,
`docker-compose.yml`, and catalog key rotation runbook
## Suggested Migration Order
1. Update local checkout path, remote, repository manifest, and docs.
2. Move Gitea helper scripts and source templates from core to meta.
3. Move release/catalog command entry points from core to meta and convert
repository discovery to `repositories.json`.
4. Move whole-product Gitea Actions workflows to meta.
5. Move security audit toolbox to meta.
6. Move/wrap whole-product Docker composition in meta.
7. Remove core compatibility wrappers once docs and workflows use `govoplan`.
## Current State
Completed in the meta repository:
- canonical tools live in `govoplan/tools`
- release/catalog/Gitea/security-audit tools have been moved from core to meta
- whole-product Gitea workflows have been moved to `govoplan/.gitea/workflows`
- shared PostgreSQL and production-like Docker profiles live under `govoplan/dev`
- core-local wrappers for moved commands have been removed
- the meta `scripts` compatibility layer has been removed; use `govoplan/tools`

133
docs/REPOSITORY_STRUCTURE.md Normal file
View File

@@ -0,0 +1,133 @@
# GovOPlaN Repository Structure
GovOPlaN uses separate repositories for runtime modules, connector modules,
system orchestration, and public website/catalog publication.
## Categories
### system
System repositories are not optional runtime modules. They provide platform
execution, orchestration, development entry points, installer tooling, or core
runtime contracts.
Current system repositories:
- `govoplan`: meta repository for whole-product orchestration.
- `govoplan-core`: runtime kernel/server runner, module registry, shared UI
shell, installer, migration orchestration, and core contracts.
### module
Runtime modules provide user-visible product behavior. They can include backend
routes, database migrations, frontend contributions, permissions, docs, workers,
and lifecycle hooks.
Modules must communicate through core contracts, capabilities, providers, and
routes. They must not import another module's internal implementation directly.
### connector
Connector repositories adapt GovOPlaN capabilities to external protocols,
platforms, and integration formats. A connector may be a runtime module, but its
business responsibility is transport/integration, not product semantics.
Examples:
- `govoplan-rest`
- `govoplan-soap`
- `govoplan-fit-connect`
- `govoplan-xoev`
- `govoplan-xrechnung`
- `govoplan-xta-osci`
- `govoplan-connectors`
### website
Website repositories are public content or publication targets. They are not
installed into a GovOPlaN runtime.
Current website repository:
- `addideas-govoplan-website`: public website and release catalog publication target.
## Meta Repository Role
`govoplan` is the canonical operator entry point. Whole-product commands should
be available here first:
- development launch
- repository bootstrap/status
- whole-product Docker composition
- release tagging
- catalog publication
- installer/daemon orchestration
- cross-repository Gitea issue/wiki/label helpers
- security and dependency audit tooling
Whole-product command implementations live under `govoplan/tools`.
Core keeps runtime-specific commands only.
Repositories are linked through `repositories.json` and scripts rather than git
submodules. Development checkouts can keep their own branches and dirty state,
while release/catalog tooling can still resolve the exact repository list from
one place. If a deployment profile later needs pinned SHAs for every repository,
generate that lock as a release artifact instead of making day-to-day
development depend on submodule updates.
## Docker Placement
Whole-product Docker and production-like deployment composition belongs in
`govoplan`.
Current shared profiles:
- `dev/postgres`
- `dev/production-like`
Website-specific serving profiles stay with `addideas-govoplan-website`.
Module-specific test beds stay in the owning module repository. Examples:
- `govoplan-files/dev/connectors`
- `govoplan-campaign/dev/mail-testbed`
- connector protocol test beds in their connector repositories
This keeps local module tests close to the code while giving operators one
repository for product deployment.
## addideas-govoplan-website and Release Logic
`addideas-govoplan-website` should be treated as the public website and catalog publication
target, not as the release orchestrator.
The release workflow should be invoked from `govoplan`. It may write signed
catalog artifacts into `addideas-govoplan-website/public/catalogs/...` and then build or
deploy the website.
A separate `govoplan-release` repository should only be introduced if release
logic becomes an independently versioned service or toolchain. For now, a
separate repository would add coordination overhead without a clear boundary.
If embeddable public widgets become a runtime capability later, create a
separate installable module for that. The public website repository should stay
website/publication content only.
## Config Import/Export
Configuration import/export is runtime administration behavior. It should stay
with core/admin until it grows into a separately installable product capability.
Create a `govoplan-config` module only if configuration packages need their own
module lifecycle, permissions, migrations, background jobs, or public API.
## No govoplan-meta Runtime Module
Do not add a `govoplan-meta` runtime module for repository metadata. Repository
metadata belongs in this meta repository. Installed runtime metadata belongs in
the core module registry and module manifests.
If a runtime "system" surface is needed, it should be a core/admin/ops feature
backed by explicit capabilities, not a pseudo-module that mirrors repository
state.

143
docs/SECURITY_AUDIT.md Normal file
View File

@@ -0,0 +1,143 @@
# Security Audit Toolchain
GovOPlaN uses a free/open-source-first audit toolchain that can run locally,
inside a container, and in Gitea Actions.
## Tools
- Semgrep: multi-language SAST, with GovOPlaN-specific local rules plus
explicit public registry rulesets in CI/full runs.
- Bandit: Python AST security checks.
- Ruff `S` rules: fast flake8-bandit-compatible Python security linting.
- Gitleaks: committed-secret scanning.
- Trivy: filesystem dependency, secret, and misconfiguration scanning.
- pip-audit and npm audit: package vulnerability scanning from dependency
manifests/locks.
- OSV-Scanner: recursive dependency vulnerability scan in full mode.
- jscpd: duplicated-code reports in full mode.
- Radon/Xenon: Python complexity reports and thresholds in full mode.
## Local Usage
Build and run the toolbox from this repository:
```bash
cd /mnt/DATA/git/govoplan
tools/checks/security-audit/run.sh --mode ci --scope current
```
Scan all sibling GovOPlaN repositories under `/mnt/DATA/git`:
```bash
cd /mnt/DATA/git/govoplan
tools/checks/security-audit/run.sh --mode full --scope govoplan
```
Reports are written to `audit-reports/`, which is intentionally ignored by git.
The wrapper tags the toolbox image by a fingerprint of the Dockerfile and
`requirements-audit.txt`. If those inputs have not changed, subsequent runs reuse
the existing local image instead of reinstalling all tools. The stable alias is
`govoplan/security-audit:local` unless `SECURITY_AUDIT_IMAGE` is set.
Force a cached rebuild:
```bash
tools/checks/security-audit/run.sh --mode ci --scope current --rebuild
```
Refresh from upstream base images and package ranges:
```bash
tools/checks/security-audit/run.sh --mode ci --scope current --update
```
Build or refresh the toolbox without running an audit:
```bash
tools/checks/security-audit/run.sh --mode quick --scope current --build-only
tools/checks/security-audit/run.sh --mode quick --scope current --update --build-only
```
## Modes
- `quick`: local Semgrep rules, Bandit, Ruff security rules, Gitleaks.
- `ci`: quick plus Semgrep public registry rulesets, Trivy, pip-audit, npm audit.
- `full`: ci plus OSV-Scanner, jscpd, Radon, and Xenon.
## Gating
The initial Gitea workflow runs in report-only mode:
```bash
SECURITY_AUDIT_FAIL_ON_FINDINGS=0
```
This avoids blocking every push while the first baseline is reviewed. After the
baseline is clean, switch the workflow to:
```bash
SECURITY_AUDIT_FAIL_ON_FINDINGS=1
```
or run locally with:
```bash
tools/checks/security-audit/run.sh --mode ci --scope current --strict
```
## Audit Burndown Workflow
Treat Gitea issues as the active audit state. A full GovOPlaN audit should
produce one tracker issue in `add-ideas/govoplan` and child issues in the
repository that owns each fix.
Use the tracker issue for:
- report path, timestamp, mode, and scope
- scanner counts by category
- clean scanners and resolved findings
- links to child issues
- the next audit run target
Use child issues for concrete code or configuration changes. Apply
`source/security-audit` to every issue created from a report, then add the
most specific audit label:
- `audit/quick-fix`: narrow direct remediation
- `audit/structural`: behavior or architecture needs review
- `audit/complexity`: Radon/Xenon maintainability finding
- `audit/duplication`: jscpd duplication finding
- `audit/false-positive`: reviewed narrow false positive or accepted risk
- `audit/needs-design`: human decision needed before implementation
Keep active implementation status in issues instead of committing generated
audit reports. `audit-reports/` is ignored; quote the report directory and the
important scanner counts in the tracker issue.
## Image Freshness
The regular `Security Audit` workflow reuses the fingerprinted toolbox image
when the Docker daemon is persistent, which is the normal case for the
self-hosted Gitea runner using the host Docker socket. The separate
`Security Audit Toolbox Update` workflow runs weekly with
`SECURITY_AUDIT_UPDATE=1`; it pulls current base images and re-resolves the
allowed tool version ranges into a refreshed local image.
## Direct Host Usage
The container is the recommended path. For direct host usage, install the Python
tools first:
```bash
cd /mnt/DATA/git/govoplan
python -m venv .venv
./.venv/bin/python -m pip install -r requirements-audit.txt
```
Then install the non-Python tools (`gitleaks`, `trivy`, `osv-scanner`, `jscpd`)
through the host package manager or vendor instructions and run:
```bash
tools/checks/check-security-audit.sh --mode quick --scope current
```

458
docs/gitea-labels.json Normal file
View File

@@ -0,0 +1,458 @@
[
{
"name": "type/bug",
"color": "d73a4a",
"description": "A reproducible defect, regression, or incorrect behavior.",
"exclusive": true
},
{
"name": "type/feature",
"color": "0e8a16",
"description": "New user-visible behavior or platform capability.",
"exclusive": true
},
{
"name": "type/task",
"color": "1d76db",
"description": "Implementation, maintenance, migration, or operational work.",
"exclusive": true
},
{
"name": "type/debt",
"color": "fbca04",
"description": "Cleanup, refactoring, risk reduction, or deferred engineering work.",
"exclusive": true
},
{
"name": "type/docs",
"color": "5319e7",
"description": "Documentation, process, or developer workflow work.",
"exclusive": true
},
{
"name": "priority/p0",
"color": "b60205",
"description": "Immediate stop-the-line priority.",
"exclusive": true
},
{
"name": "priority/p1",
"color": "d93f0b",
"description": "High priority for the next focused work window.",
"exclusive": true
},
{
"name": "priority/p2",
"color": "fbca04",
"description": "Normal planned priority.",
"exclusive": true
},
{
"name": "priority/p3",
"color": "c2e0c6",
"description": "Low priority or opportunistic cleanup.",
"exclusive": true
},
{
"name": "status/triage",
"color": "d4c5f9",
"description": "Needs review, ownership, priority, or acceptance criteria.",
"exclusive": true
},
{
"name": "status/ready",
"color": "0e8a16",
"description": "Ready for implementation.",
"exclusive": true
},
{
"name": "status/in-progress",
"color": "0052cc",
"description": "Currently being worked.",
"exclusive": true
},
{
"name": "status/blocked",
"color": "b60205",
"description": "Cannot progress without a decision, dependency, credential, or external change.",
"exclusive": true
},
{
"name": "status/needs-info",
"color": "f9d0c4",
"description": "Needs clarifying input before implementation can proceed safely.",
"exclusive": true
},
{
"name": "module/access",
"color": "0e8a16",
"description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.",
"exclusive": false
},
{
"name": "module/addresses",
"color": "5319e7",
"description": "GovOPlaN Addresses module behavior or integration.",
"exclusive": false
},
{
"name": "module/admin",
"color": "006b75",
"description": "GovOPlaN Admin module behavior or integration.",
"exclusive": false
},
{
"name": "module/appointments",
"color": "d876e3",
"description": "GovOPlaN Appointments module behavior or integration.",
"exclusive": false
},
{
"name": "module/audit",
"color": "0e8a16",
"description": "GovOPlaN Audit module behavior or integration.",
"exclusive": false
},
{
"name": "module/calendar",
"color": "bfd4f2",
"description": "GovOPlaN Calendar module behavior or integration.",
"exclusive": false
},
{
"name": "module/campaign",
"color": "d876e3",
"description": "GovOPlaN campaign module behavior or integration.",
"exclusive": false
},
{
"name": "module/cases",
"color": "1d76db",
"description": "GovOPlaN Cases module behavior or integration.",
"exclusive": false
},
{
"name": "module/connectors",
"color": "c2e0c6",
"description": "GovOPlaN Connectors module behavior or integration.",
"exclusive": false
},
{
"name": "module/core",
"color": "0052cc",
"description": "GovOPlaN core runner, shared primitives, shell, or extension points.",
"exclusive": false
},
{
"name": "module/dms",
"color": "c5def5",
"description": "GovOPlaN Dms module behavior or integration.",
"exclusive": false
},
{
"name": "module/erp",
"color": "fef2c0",
"description": "GovOPlaN Erp module behavior or integration.",
"exclusive": false
},
{
"name": "module/files",
"color": "006b75",
"description": "GovOPlaN files module behavior or integration.",
"exclusive": false
},
{
"name": "module/fit-connect",
"color": "fbca04",
"description": "GovOPlaN Fit Connect module behavior or integration.",
"exclusive": false
},
{
"name": "module/forms",
"color": "f9d0c4",
"description": "GovOPlaN Forms module behavior or integration.",
"exclusive": false
},
{
"name": "module/identity-trust",
"color": "d4c5f9",
"description": "GovOPlaN Identity Trust module behavior or integration.",
"exclusive": false
},
{
"name": "module/identity",
"color": "bfd4f2",
"description": "GovOPlaN Identity module behavior or integration.",
"exclusive": false
},
{
"name": "module/idm",
"color": "0052cc",
"description": "GovOPlaN Idm module behavior or integration.",
"exclusive": false
},
{
"name": "module/ledger",
"color": "5319e7",
"description": "GovOPlaN Ledger module behavior or integration.",
"exclusive": false
},
{
"name": "module/mail",
"color": "5319e7",
"description": "GovOPlaN mail module behavior or integration.",
"exclusive": false
},
{
"name": "module/notifications",
"color": "d876e3",
"description": "GovOPlaN Notifications module behavior or integration.",
"exclusive": false
},
{
"name": "module/ops",
"color": "0e8a16",
"description": "GovOPlaN Ops module behavior or integration.",
"exclusive": false
},
{
"name": "module/organizations",
"color": "bfdadc",
"description": "GovOPlaN Organizations module behavior or integration.",
"exclusive": false
},
{
"name": "module/payments",
"color": "bfd4f2",
"description": "GovOPlaN Payments module behavior or integration.",
"exclusive": false
},
{
"name": "module/policy",
"color": "d93f0b",
"description": "GovOPlaN Policy module behavior or integration.",
"exclusive": false
},
{
"name": "module/portal",
"color": "1d76db",
"description": "GovOPlaN Portal module behavior or integration.",
"exclusive": false
},
{
"name": "module/postbox",
"color": "d93f0b",
"description": "GovOPlaN Postbox module behavior or integration.",
"exclusive": false
},
{
"name": "module/reporting",
"color": "c2e0c6",
"description": "GovOPlaN Reporting module behavior or integration.",
"exclusive": false
},
{
"name": "module/search",
"color": "bfdadc",
"description": "GovOPlaN Search module behavior or integration.",
"exclusive": false
},
{
"name": "module/scheduling",
"color": "d93f0b",
"description": "GovOPlaN Scheduling module behavior or integration.",
"exclusive": false
},
{
"name": "module/tasks",
"color": "c5def5",
"description": "GovOPlaN Tasks module behavior or integration.",
"exclusive": false
},
{
"name": "module/templates",
"color": "fef2c0",
"description": "GovOPlaN Templates module behavior or integration.",
"exclusive": false
},
{
"name": "module/tenancy",
"color": "e4e669",
"description": "GovOPlaN Tenancy module behavior or integration.",
"exclusive": false
},
{
"name": "module/workflow",
"color": "f9d0c4",
"description": "GovOPlaN Workflow module behavior or integration.",
"exclusive": false
},
{
"name": "module/xoev",
"color": "d4c5f9",
"description": "GovOPlaN Xoev module behavior or integration.",
"exclusive": false
},
{
"name": "module/xrechnung",
"color": "0052cc",
"description": "GovOPlaN Xrechnung module behavior or integration.",
"exclusive": false
},
{
"name": "module/xta-osci",
"color": "5319e7",
"description": "GovOPlaN Xta Osci module behavior or integration.",
"exclusive": false
},
{
"name": "area/auth",
"color": "bfd4f2",
"description": "Authentication, sessions, access bootstrap, or login behavior.",
"exclusive": false
},
{
"name": "area/tenancy",
"color": "bfdadc",
"description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.",
"exclusive": false
},
{
"name": "area/rbac",
"color": "c5def5",
"description": "Permissions, roles, delegation, or authorization policy.",
"exclusive": false
},
{
"name": "area/governance",
"color": "fef2c0",
"description": "Governance policy, audit, privacy, retention, or compliance behavior.",
"exclusive": false
},
{
"name": "area/module-system",
"color": "bfe5bf",
"description": "Module discovery, manifests, capabilities, routing, or optional integrations.",
"exclusive": false
},
{
"name": "area/migrations",
"color": "e4e669",
"description": "Alembic migrations, schema bootstrap, or persistence evolution.",
"exclusive": false
},
{
"name": "area/webui",
"color": "1d76db",
"description": "Shared WebUI shell, frontend components, routing, or frontend tests.",
"exclusive": false
},
{
"name": "area/api",
"color": "5319e7",
"description": "HTTP API contracts, routers, schemas, or API smoke behavior.",
"exclusive": false
},
{
"name": "area/db",
"color": "0e8a16",
"description": "Database sessions, models, transactions, or persistence primitives.",
"exclusive": false
},
{
"name": "area/devex",
"color": "bfd4f2",
"description": "Local developer workflow, scripts, tests, tooling, or release helpers.",
"exclusive": false
},
{
"name": "area/release",
"color": "c2e0c6",
"description": "Versioning, release locks, tags, packaging, or dependency pins.",
"exclusive": false
},
{
"name": "area/security",
"color": "b60205",
"description": "Security posture, static analysis, supply-chain hardening, or vulnerability remediation.",
"exclusive": false
},
{
"name": "area/docs",
"color": "5319e7",
"description": "Durable documentation and project guidance.",
"exclusive": false
},
{
"name": "area/marketing",
"color": "f9d0c4",
"description": "Public website, product messaging, publication copy, or legal page content.",
"exclusive": false
},
{
"name": "source/todo-scan",
"color": "cccccc",
"description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.",
"exclusive": false
},
{
"name": "source/backlog-import",
"color": "bfdadc",
"description": "Imported from markdown backlog, roadmap, plan, or TODO files.",
"exclusive": false
},
{
"name": "source/security-audit",
"color": "d4c5f9",
"description": "Created from a structured security or code-quality audit report.",
"exclusive": false
},
{
"name": "audit/quick-fix",
"color": "0e8a16",
"description": "Audit finding that appears narrow and directly fixable.",
"exclusive": false
},
{
"name": "audit/structural",
"color": "d93f0b",
"description": "Audit finding that needs design, refactoring, or behavior review.",
"exclusive": false
},
{
"name": "audit/complexity",
"color": "fbca04",
"description": "Complexity finding from Radon, Xenon, or equivalent maintainability scans.",
"exclusive": false
},
{
"name": "audit/duplication",
"color": "c2e0c6",
"description": "Duplicated-code finding from jscpd or equivalent similarity scans.",
"exclusive": false
},
{
"name": "audit/false-positive",
"color": "cccccc",
"description": "Audit finding reviewed as a narrow false positive or acceptable risk.",
"exclusive": false
},
{
"name": "audit/needs-design",
"color": "f9d0c4",
"description": "Audit finding that needs an architectural or product decision before implementation.",
"exclusive": false
},
{
"name": "codex/ready",
"color": "0e8a16",
"description": "Suitable for Codex to pick up with the existing issue context.",
"exclusive": false
},
{
"name": "codex/needs-human",
"color": "f9d0c4",
"description": "Needs an explicit human decision before Codex should implement.",
"exclusive": false
}
]

70
repositories.json Normal file
View File

@@ -0,0 +1,70 @@
{
"version": 1,
"organization": "add-ideas",
"default_parent": "/mnt/DATA/git",
"repositories": [
{"name": "govoplan", "category": "system", "subtype": "meta", "remote": "git@git.add-ideas.de:add-ideas/govoplan.git", "path": "govoplan"},
{"name": "govoplan-core", "category": "system", "subtype": "kernel", "remote": "git@git.add-ideas.de:add-ideas/govoplan-core.git", "path": "govoplan-core"},
{"name": "govoplan-access", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-access.git", "path": "govoplan-access"},
{"name": "govoplan-addresses", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-addresses.git", "path": "govoplan-addresses"},
{"name": "govoplan-admin", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-admin.git", "path": "govoplan-admin"},
{"name": "govoplan-appointments", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-appointments.git", "path": "govoplan-appointments"},
{"name": "govoplan-approvals", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-approvals.git", "path": "govoplan-approvals"},
{"name": "govoplan-assets", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-assets.git", "path": "govoplan-assets"},
{"name": "govoplan-audit", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-audit.git", "path": "govoplan-audit"},
{"name": "govoplan-booking", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-booking.git", "path": "govoplan-booking"},
{"name": "govoplan-calendar", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-calendar.git", "path": "govoplan-calendar"},
{"name": "govoplan-campaign", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-campaign.git", "path": "govoplan-campaign"},
{"name": "govoplan-cases", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-cases.git", "path": "govoplan-cases"},
{"name": "govoplan-certificates", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-certificates.git", "path": "govoplan-certificates"},
{"name": "govoplan-committee", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-committee.git", "path": "govoplan-committee"},
{"name": "govoplan-connectors", "category": "connector", "subtype": "connector-hub", "remote": "git@git.add-ideas.de:add-ideas/govoplan-connectors.git", "path": "govoplan-connectors"},
{"name": "govoplan-consultation", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-consultation.git", "path": "govoplan-consultation"},
{"name": "govoplan-contracts", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-contracts.git", "path": "govoplan-contracts"},
{"name": "govoplan-dashboard", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-dashboard.git", "path": "govoplan-dashboard"},
{"name": "govoplan-dms", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-dms.git", "path": "govoplan-dms"},
{"name": "govoplan-docs", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-docs.git", "path": "govoplan-docs"},
{"name": "govoplan-erp", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-erp.git", "path": "govoplan-erp"},
{"name": "govoplan-facilities", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-facilities.git", "path": "govoplan-facilities"},
{"name": "govoplan-files", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-files.git", "path": "govoplan-files"},
{"name": "govoplan-fit-connect", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-fit-connect.git", "path": "govoplan-fit-connect"},
{"name": "govoplan-forms", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-forms.git", "path": "govoplan-forms"},
{"name": "govoplan-forms-runtime", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-forms-runtime.git", "path": "govoplan-forms-runtime"},
{"name": "govoplan-grants", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-grants.git", "path": "govoplan-grants"},
{"name": "govoplan-helpdesk", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-helpdesk.git", "path": "govoplan-helpdesk"},
{"name": "govoplan-identity", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-identity.git", "path": "govoplan-identity"},
{"name": "govoplan-identity-trust", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-identity-trust.git", "path": "govoplan-identity-trust"},
{"name": "govoplan-idm", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-idm.git", "path": "govoplan-idm"},
{"name": "govoplan-inspections", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-inspections.git", "path": "govoplan-inspections"},
{"name": "govoplan-issue-reporting", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-issue-reporting.git", "path": "govoplan-issue-reporting"},
{"name": "govoplan-learning", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-learning.git", "path": "govoplan-learning"},
{"name": "govoplan-ledger", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-ledger.git", "path": "govoplan-ledger"},
{"name": "govoplan-mail", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-mail.git", "path": "govoplan-mail"},
{"name": "govoplan-notifications", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-notifications.git", "path": "govoplan-notifications"},
{"name": "govoplan-ops", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-ops.git", "path": "govoplan-ops"},
{"name": "govoplan-organizations", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-organizations.git", "path": "govoplan-organizations"},
{"name": "govoplan-payments", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-payments.git", "path": "govoplan-payments"},
{"name": "govoplan-permits", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-permits.git", "path": "govoplan-permits"},
{"name": "govoplan-policy", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-policy.git", "path": "govoplan-policy"},
{"name": "govoplan-portal", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-portal.git", "path": "govoplan-portal"},
{"name": "govoplan-postbox", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-postbox.git", "path": "govoplan-postbox"},
{"name": "govoplan-procurement", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-procurement.git", "path": "govoplan-procurement"},
{"name": "govoplan-records", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-records.git", "path": "govoplan-records"},
{"name": "govoplan-reporting", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-reporting.git", "path": "govoplan-reporting"},
{"name": "govoplan-resources", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-resources.git", "path": "govoplan-resources"},
{"name": "govoplan-rest", "category": "connector", "subtype": "protocol", "remote": "git@git.add-ideas.de:add-ideas/govoplan-rest.git", "path": "govoplan-rest"},
{"name": "govoplan-risk-compliance", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-risk-compliance.git", "path": "govoplan-risk-compliance"},
{"name": "govoplan-scheduling", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-scheduling.git", "path": "govoplan-scheduling"},
{"name": "govoplan-search", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-search.git", "path": "govoplan-search"},
{"name": "govoplan-soap", "category": "connector", "subtype": "protocol", "remote": "git@git.add-ideas.de:add-ideas/govoplan-soap.git", "path": "govoplan-soap"},
{"name": "govoplan-tasks", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-tasks.git", "path": "govoplan-tasks"},
{"name": "govoplan-templates", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-templates.git", "path": "govoplan-templates"},
{"name": "govoplan-tenancy", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-tenancy.git", "path": "govoplan-tenancy"},
{"name": "govoplan-transparency", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:add-ideas/govoplan-transparency.git", "path": "govoplan-transparency"},
{"name": "addideas-govoplan-website", "category": "website", "subtype": "public-site", "remote": "git@git.add-ideas.de:add-ideas/addideas-govoplan-website.git", "path": "addideas-govoplan-website"},
{"name": "govoplan-workflow", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:add-ideas/govoplan-workflow.git", "path": "govoplan-workflow"},
{"name": "govoplan-xoev", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xoev.git", "path": "govoplan-xoev"},
{"name": "govoplan-xrechnung", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xrechnung.git", "path": "govoplan-xrechnung"},
{"name": "govoplan-xta-osci", "category": "connector", "subtype": "standard", "remote": "git@git.add-ideas.de:add-ideas/govoplan-xta-osci.git", "path": "govoplan-xta-osci"}
]
}

10
requirements-audit.txt Normal file
View File

@@ -0,0 +1,10 @@
bandit>=1.8,<2
filelock>=3.20.3
idna>=3.15
pip>=26.1.2
pip-audit>=2.9,<3
python-multipart>=0.0.31
radon>=6,<7
ruff>=0.14,<1
semgrep>=1.140,<2
xenon>=0.9,<1

27
requirements-dev.txt Normal file
View File

@@ -0,0 +1,27 @@
# Whole-product editable development environment.
#
# This file is intentionally owned by the meta repository because it wires the
# sibling core/module checkouts into one local GovOPlaN workspace. Package-local
# runtime dependencies stay in the owning repositories' pyproject.toml files.
-e ../govoplan-core[server]
-e ../govoplan-tenancy
-e ../govoplan-organizations
-e ../govoplan-identity
-e ../govoplan-access
-e ../govoplan-admin
-e ../govoplan-policy
-e ../govoplan-audit
-e ../govoplan-dashboard
-e ../govoplan-files
-e ../govoplan-mail
-e ../govoplan-campaign
-e ../govoplan-calendar
-e ../govoplan-docs
-e ../govoplan-ops
httpx==0.28.1
httpx2>=2.5,<3
filelock>=3.20.3
idna>=3.15
pip>=26.1.2
pip-audit>=2.9,<3
python-multipart>=0.0.31

19
requirements-release.txt Normal file
View File

@@ -0,0 +1,19 @@
# Whole-product release install from tagged module repositories.
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when
# cutting a release.
../govoplan-core[server]
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
govoplan-idm @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-idm.git@v0.1.6
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6

40
tools/checks/check-dependency-audits.sh Normal file
View File

@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
NODE_BIN="$(dirname "$NPM")"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
if [[ ! -x "$NPM" ]]; then
NPM=npm
NODE_BIN="$(dirname "$(command -v "$NPM")")"
fi
cd "$ROOT"
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$META_ROOT/tools/checks/check-dependency-hygiene.sh"
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
echo "pip-audit is not installed. Install dev requirements first:" >&2
echo " $PYTHON -m pip install -r $META_ROOT/requirements-dev.txt" >&2
exit 127
fi
python_status=0
npm_status=0
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
cd "$ROOT/webui"
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
exit 1
fi

162
tools/checks/check-dependency-hygiene.sh Normal file
View File

@@ -0,0 +1,162 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
cd "$ROOT"
"$PYTHON" - <<'PY'
import importlib.util
import sys
if importlib.util.find_spec("pip") is None:
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
print(" cd /mnt/DATA/git/govoplan", file=sys.stderr)
print(" python3 -m venv .venv", file=sys.stderr)
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
raise SystemExit(127)
PY
"$PYTHON" -m pip check
"$PYTHON" - <<'PY'
from __future__ import annotations
import importlib.metadata
import pathlib
import sys
prefix = pathlib.Path(sys.prefix)
legacy_patterns = (
"__editable__.govoplan_module_multimailer-*.pth",
"__editable___govoplan_module_multimailer_*_finder.py",
"govoplan_module_multimailer-*.dist-info",
)
problems: list[pathlib.Path] = []
for site_packages in (prefix / "lib").glob("python*/site-packages"):
for pattern in legacy_patterns:
problems.extend(site_packages.glob(pattern))
for dist in importlib.metadata.distributions():
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
problems.append(pathlib.Path(str(dist.locate_file(""))))
if problems:
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
for path in sorted(set(problems)):
print(f" {path}", file=sys.stderr)
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
raise SystemExit(1)
print("No stale legacy multimailer package metadata found.")
PY
"$PYTHON" - <<'PY'
from __future__ import annotations
import pathlib
import sys
root = pathlib.Path.cwd()
scan_roots = [
root / "src",
root / "tests",
root.parent / "govoplan-access" / "src",
root.parent / "govoplan-admin" / "src",
root.parent / "govoplan-audit" / "src",
root.parent / "govoplan-calendar" / "src",
root.parent / "govoplan-campaign" / "src",
root.parent / "govoplan-dashboard" / "src",
root.parent / "govoplan-files" / "src",
root.parent / "govoplan-identity" / "src",
root.parent / "govoplan-mail" / "src",
root.parent / "govoplan-ops" / "src",
root.parent / "govoplan-organizations" / "src",
root.parent / "govoplan-policy" / "src",
root.parent / "govoplan-tenancy" / "src",
]
deprecated_tokens = {
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
}
hits: list[str] = []
for scan_root in scan_roots:
if not scan_root.exists():
continue
for path in scan_root.rglob("*.py"):
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
continue
for token, replacement in deprecated_tokens.items():
if token in text:
hits.append(f"{path}: deprecated {token}. {replacement}")
if hits:
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
print("\n".join(hits), file=sys.stderr)
raise SystemExit(1)
print("Deprecated framework constant scan passed.")
PY
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
0|false|False|no|No)
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
;;
auto)
if "$PYTHON" - <<'PY'
import importlib.util
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
PY
then
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
else
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
fi
;;
*)
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
;;
esac
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
"$PYTHON" - <<'PY'
import warnings
from starlette.exceptions import StarletteDeprecationWarning
warnings.simplefilter("error", StarletteDeprecationWarning)
from fastapi import FastAPI
from fastapi.testclient import TestClient
app = FastAPI()
@app.get("/dependency-hygiene-ping")
def ping() -> dict[str, bool]:
return {"ok": True}
with TestClient(app) as client:
response = client.get("/dependency-hygiene-ping")
assert response.status_code == 200
assert response.json() == {"ok": True}
print("TestClient deprecation smoke passed.")
PY
fi
echo "Dependency hygiene check passed."

86
tools/checks/check-focused.sh Normal file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
NPM="$NODE/npm"
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
trap 'rm -f "$NPM_USERCONFIG"' EXIT
[ -x "$PYTHON" ] || {
echo "check-focused: Python virtualenv not found at $PYTHON." >&2
echo "Run: cd $META_ROOT && python3 -m venv .venv && ./.venv/bin/python -m pip install --upgrade pip && ./.venv/bin/python -m pip install -r requirements-dev.txt" >&2
exit 127
}
export PATH="$WEBUI_BIN:$NODE:$PATH"
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
unset npm_config_tmp NPM_CONFIG_TMP
cd "$ROOT"
GOVOPLAN_CORE_ROOT="$ROOT" PYTHON="$PYTHON" CHECK_TESTCLIENT_DEPRECATIONS=1 bash "$META_ROOT/tools/checks/check-dependency-hygiene.sh"
"$PYTHON" - <<'PY'
import ast
import pathlib
import sys
roots = [
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
pathlib.Path("/mnt/DATA/git/govoplan-mail/tests"),
]
errors = []
count = 0
for root in roots:
if not root.exists():
continue
for path in root.rglob("*.py"):
count += 1
try:
ast.parse(path.read_text(), filename=str(path))
except SyntaxError as exc:
errors.append(f"{path}:{exc.lineno}:{exc.offset}: {exc.msg}")
if errors:
print("\n".join(errors))
sys.exit(1)
print(f"AST syntax check passed for {count} Python files")
PY
"$PYTHON" -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
"$META_ROOT/tools/checks/check_dependency_boundaries.py"
"$PYTHON" -m unittest tests.test_module_system
"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
"$PYTHON" -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
cd "$ROOT/webui"
"$NPM" run test:mail-components
"$NPM" run test:module-capabilities
"$NPM" run test:module-permutations
cd /mnt/DATA/git/govoplan-mail/webui
"$NPM" run test:mail-ui
cd /mnt/DATA/git/govoplan-campaign/webui
"$NPM" run test:policy-ui
"$NPM" run test:template-preview

26
tools/checks/check-module-matrix.sh Normal file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
NODE_BIN="$(dirname "$NPM")"
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
if [[ ! -x "$NPM" ]]; then
NPM=npm
NODE_BIN="$(dirname "$(command -v "$NPM")")"
fi
cd "$ROOT"
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
"$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py"
cd "$ROOT/webui"
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations

283
tools/checks/check-release-integration.sh Normal file
View File

@@ -0,0 +1,283 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
NODE_BIN="$(dirname "$NPM")"
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-integration.XXXXXXXX")"
trap 'rm -rf "$WORK_ROOT"; rm -f "$NPM_USERCONFIG"' EXIT
if [[ ! -x "$PYTHON" ]]; then
PYTHON=python
fi
if [[ ! -x "$NPM" ]]; then
NPM=npm
NODE_BIN="$(dirname "$(command -v "$NPM")")"
fi
cd "$ROOT"
RELEASE_VERSION="$("$PYTHON" - <<'PY'
from pathlib import Path
import tomllib
with (Path.cwd() / "pyproject.toml").open("rb") as handle:
print(tomllib.load(handle)["project"]["version"])
PY
)"
RELEASE_TAG="${GOVOPLAN_RELEASE_TAG:-v$RELEASE_VERSION}"
export RELEASE_TAG RELEASE_VERSION
export META_ROOT
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
export GOVOPLAN_MIGRATION_TRACK=release
export PATH="$WEBUI_BIN:$NODE_BIN:$PATH"
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
unset npm_config_tmp NPM_CONFIG_TMP
run_step() {
echo
echo "==> $*"
}
retry() {
local attempt status
for attempt in 1 2 3; do
if "$@"; then
return 0
fi
status=$?
if [[ "$attempt" == 3 ]]; then
return "$status"
fi
sleep $((attempt * 10))
done
}
run_discovered_tests() {
local suite_dir="$1"
local status
if ! find "$suite_dir" -type f -name 'test*.py' -print -quit | grep -q .; then
echo "No unittest test files found under $suite_dir; skipping."
return 0
fi
set +e
"$PYTHON" -m unittest discover -s "$suite_dir"
status=$?
set -e
if [[ "$status" == 5 ]]; then
echo "No unittest tests discovered under $suite_dir; skipping."
return 0
fi
return "$status"
}
install_cloned_webui_dependencies() {
local webui_dir="$1"
if [[ ! -f "$webui_dir/package.json" ]]; then
echo "No package.json found under $webui_dir; skipping WebUI dependency install."
return 0
fi
(
cd "$webui_dir"
retry "$NPM" install --prefer-online --include=dev --no-save
)
}
run_step "Validate release refs and installed package metadata"
"$PYTHON" - <<'PY'
from __future__ import annotations
import importlib.metadata as metadata
import json
import os
import re
import subprocess
import sys
from pathlib import Path
root = Path.cwd()
release_requirements = Path(os.environ["META_ROOT"]) / "requirements-release.txt"
release_webui = root / "webui" / "package.release.json"
expected_version = os.environ["RELEASE_VERSION"]
expected_tag = os.environ["RELEASE_TAG"]
errors: list[str] = []
python_requirements: list[tuple[str, str, str]] = []
for line in release_requirements.read_text(encoding="utf-8").splitlines():
match = re.match(r"^(govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://(.+?\.git)@(v[0-9]+\.[0-9]+\.[0-9]+)$", line.strip())
if match:
python_requirements.append((match.group(1), f"ssh://{match.group(2)}", match.group(3)))
if not python_requirements:
errors.append("No GovOPlaN git+ssh release requirements found.")
for package_name, repo_url, tag in python_requirements:
if tag != expected_tag:
errors.append(f"{package_name} release requirement uses {tag!r}, expected {expected_tag!r}.")
version = metadata.version(package_name)
if version != tag.removeprefix("v"):
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
result = subprocess.run(
["git", "ls-remote", "--tags", repo_url, f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=45,
check=False,
)
if result.returncode != 0 or f"refs/tags/{tag}" not in result.stdout:
errors.append(f"{package_name} tag {tag} is not readable from {repo_url}: {result.stderr.strip()}")
release_package = json.loads(release_webui.read_text(encoding="utf-8"))
if release_package.get("version") != expected_version:
errors.append(f"Core release WebUI version is {release_package.get('version')!r}, expected {expected_version!r}.")
webui_dependencies = release_package.get("dependencies", {})
for package_name, spec in sorted(webui_dependencies.items()):
if not package_name.startswith("@govoplan/"):
continue
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
if tag != expected_tag:
errors.append(f"{package_name} release dependency uses {tag!r}, expected {expected_tag}.")
package_json = root / "webui" / "node_modules" / package_name / "package.json"
if not package_json.exists():
errors.append(f"{package_name} is missing from release node_modules.")
continue
installed = json.loads(package_json.read_text(encoding="utf-8"))
if installed.get("version") != expected_version:
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {expected_version!r}.")
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
root_peers = release_package.get("peerDependencies", {})
for package_json in sorted((root / "webui" / "node_modules" / "@govoplan").glob("*/package.json")):
package = json.loads(package_json.read_text(encoding="utf-8"))
peers = package.get("peerDependencies", {})
for name in shared_peers:
if name in peers and name in root_peers and peers[name] != root_peers[name]:
errors.append(f"{package.get('name')} peer {name}={peers[name]!r}, expected {root_peers[name]!r}.")
if errors:
print("\n".join(errors), file=sys.stderr)
raise SystemExit(1)
print(f"Release refs and metadata passed for {len(python_requirements)} Python packages and {len(webui_dependencies)} WebUI packages.")
PY
run_step "Validate installed module manifests and registry"
"$PYTHON" - <<'PY'
from __future__ import annotations
import sys
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
expected = {
"access",
"admin",
"audit",
"calendar",
"campaigns",
"dashboard",
"docs",
"files",
"identity",
"idm",
"mail",
"ops",
"organizations",
"policy",
"tenancy",
}
manifests = available_module_manifests()
missing = sorted(expected - set(manifests))
if missing:
print(f"Missing installed module manifests: {', '.join(missing)}", file=sys.stderr)
raise SystemExit(1)
registry = build_platform_registry(tuple(sorted(expected)))
snapshot = registry.validate()
print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests))
PY
run_step "Run SQLite migration smoke for release modules"
"$PYTHON" - <<'PY'
from __future__ import annotations
import tempfile
from pathlib import Path
from govoplan_core.db.migrations import migrate_database
enabled_modules = (
"tenancy",
"organizations",
"identity",
"idm",
"access",
"admin",
"dashboard",
"policy",
"audit",
"campaigns",
"files",
"mail",
"calendar",
"docs",
"ops",
)
with tempfile.TemporaryDirectory(prefix="govoplan-release-migration-") as tmp:
database_url = f"sqlite:///{Path(tmp) / 'release-integration.sqlite3'}"
result = migrate_database(database_url=database_url, enabled_modules=enabled_modules)
if not result.current_revision:
raise SystemExit("Migration smoke did not produce an Alembic revision.")
print(f"SQLite release migration smoke reached {result.current_revision}.")
PY
run_step "Clone release module test sources"
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
git clone --depth 1 --branch "$RELEASE_TAG" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
done
run_step "Run core backend release tests"
"$PYTHON" -m unittest \
tests.test_module_system \
tests.test_access_contracts \
tests.test_identity_organization_contracts \
tests.test_core_events \
tests.test_policy_contracts
"$PYTHON" "$META_ROOT/tools/checks/check_dependency_boundaries.py"
run_step "Run cloned module backend tests"
run_discovered_tests "$WORK_ROOT/govoplan-mail/tests"
run_discovered_tests "$WORK_ROOT/govoplan-calendar/tests"
run_discovered_tests "$WORK_ROOT/govoplan-campaign/tests"
run_step "Run core WebUI release tests and build"
cd "$ROOT/webui"
"$NPM" run test:mail-components
"$NPM" run test:module-capabilities
"$NPM" run test:module-permutations
"$NPM" run build
run_step "Run cloned module WebUI tests"
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-mail/webui"
cd "$WORK_ROOT/govoplan-mail/webui"
"$NPM" run test:mail-ui
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-campaign/webui"
cd "$WORK_ROOT/govoplan-campaign/webui"
"$NPM" run test:policy-ui
"$NPM" run test:template-preview
"$NPM" run test:import-utils
echo
echo "Release integration check passed."

367
tools/checks/check-security-audit.sh Normal file
View File

@@ -0,0 +1,367 @@
#!/usr/bin/env bash
set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODE="${SECURITY_AUDIT_MODE:-ci}"
SCOPE="${SECURITY_AUDIT_SCOPE:-current}"
REPORTS_DIR="${SECURITY_AUDIT_REPORTS_DIR:-$ROOT/audit-reports}"
FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}"
REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}"
while [[ $# -gt 0 ]]; do
case "$1" in
--mode)
MODE="${2:?missing mode}"
shift 2
;;
--scope)
SCOPE="${2:?missing scope}"
shift 2
;;
--reports-dir)
REPORTS_DIR="${2:?missing reports dir}"
shift 2
;;
--strict)
FAIL_ON_FINDINGS=1
shift
;;
--report-only)
FAIL_ON_FINDINGS=0
shift
;;
-h|--help)
cat <<'EOF'
Usage: tools/checks/check-security-audit.sh [--mode ci|quick|full] [--scope current|govoplan] [--reports-dir DIR] [--strict|--report-only]
Runs GovOPlaN static/security audit tools and writes machine-readable reports.
Modes:
quick Semgrep local rules, Bandit, Ruff security rules, Gitleaks
ci quick + Semgrep registry auto config, Trivy, dependency manifest scans
full ci + OSV-Scanner, jscpd duplicate scan, Radon/Xenon complexity reports
Scopes:
current scan this repository checkout
govoplan scan sibling govoplan-* repositories below GOVOPLAN_REPOS_ROOT or the parent of this repo
By default findings are reported but do not fail the process. Use --strict or
SECURITY_AUDIT_FAIL_ON_FINDINGS=1 once the baseline is clean.
EOF
exit 0
;;
*)
echo "Unknown argument: $1" >&2
exit 2
;;
esac
done
case "$MODE" in
quick|ci|full) ;;
*)
echo "Unknown SECURITY_AUDIT_MODE=$MODE; expected quick, ci, or full." >&2
exit 2
;;
esac
case "$SCOPE" in
current|govoplan) ;;
*)
echo "Unknown SECURITY_AUDIT_SCOPE=$SCOPE; expected current or govoplan." >&2
exit 2
;;
esac
if [[ "$REPORTS_DIR" != /* ]]; then
REPORTS_DIR="$ROOT/$REPORTS_DIR"
fi
mkdir -p "$REPORTS_DIR"
declare -a REPOS=()
if [[ "$SCOPE" == "govoplan" ]]; then
REPOS_ROOT="${GOVOPLAN_REPOS_ROOT:-$(dirname "$ROOT")}"
[[ -d "$ROOT/.git" ]] && REPOS+=("$ROOT")
while IFS= read -r repo; do
[[ -d "$repo/.git" ]] || continue
REPOS+=("$repo")
done < <(find "$REPOS_ROOT" -maxdepth 1 -type d \( -name 'govoplan-*' -o -name 'addideas-govoplan-*' \) | sort)
else
REPOS=("$ROOT")
fi
if [[ "${#REPOS[@]}" -eq 0 ]]; then
echo "No repositories found for scope $SCOPE." >&2
exit 1
fi
declare -a PY_ROOTS=()
for repo in "${REPOS[@]}"; do
[[ -d "$repo/src" ]] && PY_ROOTS+=("$repo/src")
[[ -d "$repo/tests" ]] && PY_ROOTS+=("$repo/tests")
done
safe_name() {
basename "$1" | tr -c 'A-Za-z0-9_.-' '_'
}
has_tool() {
command -v "$1" >/dev/null 2>&1
}
overall_status=0
finding_status=0
missing_status=0
run_step() {
local name="$1"
shift
echo
echo "==> $name"
"$@"
local status=$?
if [[ "$status" -ne 0 ]]; then
echo "Audit step reported findings or failed: $name (exit $status)" >&2
finding_status=1
if [[ "$FAIL_ON_FINDINGS" == "1" ]]; then
overall_status=1
fi
fi
}
skip_or_fail_missing() {
local tool="$1"
echo "Skipping $tool: command not found. Use tools/checks/security-audit/run.sh for the containerized toolbox." >&2
if [[ "$REQUIRE_TOOLS" == "1" ]]; then
missing_status=1
overall_status=1
fi
}
write_manifest() {
{
printf '{\n'
printf ' "mode": "%s",\n' "$MODE"
printf ' "scope": "%s",\n' "$SCOPE"
printf ' "fail_on_findings": "%s",\n' "$FAIL_ON_FINDINGS"
printf ' "repositories": [\n'
local index=0
for repo in "${REPOS[@]}"; do
[[ "$index" -gt 0 ]] && printf ',\n'
printf ' "%s"' "$repo"
index=$((index + 1))
done
printf '\n ]\n'
printf '}\n'
} > "$REPORTS_DIR/manifest.json"
}
run_semgrep() {
local report="$REPORTS_DIR/semgrep.sarif"
local config_args=(--config "$ROOT/tools/checks/security-audit/semgrep-govoplan.yml")
if [[ "$MODE" != "quick" ]]; then
config_args+=(--config p/default --config p/owasp-top-ten)
fi
semgrep scan \
--metrics=off \
--sarif \
--output "$report" \
--exclude node_modules \
--exclude .venv \
--exclude dist \
--exclude build \
--exclude runtime \
"${config_args[@]}" \
"${REPOS[@]}"
}
run_bandit() {
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
bandit -r "${PY_ROOTS[@]}" -f json -o "$REPORTS_DIR/bandit.json"
}
run_ruff_security() {
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
ruff check --select S --output-format json "${PY_ROOTS[@]}" > "$REPORTS_DIR/ruff-security.json"
}
run_gitleaks() {
local status=0
for repo in "${REPOS[@]}"; do
local name
name="$(safe_name "$repo")"
if gitleaks git --help >/dev/null 2>&1; then
gitleaks git \
--config "$ROOT/.gitleaks.toml" \
--report-format json \
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
--no-banner \
"$repo" || status=1
else
gitleaks detect \
--source "$repo" \
--config "$ROOT/.gitleaks.toml" \
--report-format json \
--report-path "$REPORTS_DIR/gitleaks-$name.json" \
--no-banner || status=1
fi
done
return "$status"
}
run_trivy() {
local status=0
for repo in "${REPOS[@]}"; do
local name
name="$(safe_name "$repo")"
trivy fs \
--scanners vuln,secret,misconfig \
--skip-dirs node_modules \
--skip-dirs .venv \
--skip-dirs dist \
--skip-dirs build \
--format sarif \
--output "$REPORTS_DIR/trivy-$name.sarif" \
"$repo" || status=1
done
return "$status"
}
run_pip_audit_manifests() {
local status=0
for repo in "${REPOS[@]}"; do
[[ -f "$repo/requirements.txt" ]] || continue
local name
name="$(safe_name "$repo")"
pip-audit -r "$repo/requirements.txt" --progress-spinner off --format json --output "$REPORTS_DIR/pip-audit-$name.json" || status=1
done
return "$status"
}
run_npm_audit_manifests() {
local status=0
for repo in "${REPOS[@]}"; do
[[ -f "$repo/webui/package-lock.json" ]] || continue
local name
name="$(safe_name "$repo")"
(cd "$repo/webui" && npm audit --omit=dev --json > "$REPORTS_DIR/npm-audit-$name.json") || status=1
done
return "$status"
}
run_osv_scanner() {
local status=0
for repo in "${REPOS[@]}"; do
local name
name="$(safe_name "$repo")"
if osv-scanner scan --help >/dev/null 2>&1; then
osv-scanner scan -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" || status=1
else
osv-scanner -r --format json --output "$REPORTS_DIR/osv-scanner-$name.json" "$repo" || status=1
fi
done
return "$status"
}
run_jscpd() {
jscpd \
--silent \
--min-tokens 80 \
--reporters json \
--output "$REPORTS_DIR/jscpd" \
--ignore "**/.git/**,**/node_modules/**,**/.venv/**,**/dist/**,**/build/**,**/audit-reports/**,**/package-lock.json,**/package-lock.release.json,**/generatedTranslations.ts,**/docs/gitea-labels.json" \
"${REPOS[@]}"
}
run_radon() {
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
radon cc -s -a "${PY_ROOTS[@]}" > "$REPORTS_DIR/radon-cc.txt"
}
run_xenon() {
[[ "${#PY_ROOTS[@]}" -gt 0 ]] || return 0
xenon --max-absolute C --max-modules B --max-average A "${PY_ROOTS[@]}" > "$REPORTS_DIR/xenon.txt" 2>&1
}
write_manifest
if has_tool semgrep; then
run_step "Semgrep SAST" run_semgrep
else
skip_or_fail_missing semgrep
fi
if has_tool bandit; then
run_step "Bandit Python security scan" run_bandit
else
skip_or_fail_missing bandit
fi
if has_tool ruff; then
run_step "Ruff flake8-bandit security rules" run_ruff_security
else
skip_or_fail_missing ruff
fi
if has_tool gitleaks; then
run_step "Gitleaks secret scan" run_gitleaks
else
skip_or_fail_missing gitleaks
fi
if [[ "$MODE" != "quick" ]]; then
if has_tool trivy; then
run_step "Trivy filesystem vulnerability/secret/misconfig scan" run_trivy
else
skip_or_fail_missing trivy
fi
if has_tool pip-audit; then
run_step "pip-audit requirements scan" run_pip_audit_manifests
else
skip_or_fail_missing pip-audit
fi
if has_tool npm; then
run_step "npm audit lockfile scan" run_npm_audit_manifests
else
skip_or_fail_missing npm
fi
fi
if [[ "$MODE" == "full" ]]; then
if has_tool osv-scanner; then
run_step "OSV-Scanner recursive dependency scan" run_osv_scanner
else
skip_or_fail_missing osv-scanner
fi
if has_tool jscpd; then
run_step "jscpd duplicate code scan" run_jscpd
else
skip_or_fail_missing jscpd
fi
if has_tool radon; then
run_step "Radon complexity report" run_radon
else
skip_or_fail_missing radon
fi
if has_tool xenon; then
run_step "Xenon complexity threshold scan" run_xenon
else
skip_or_fail_missing xenon
fi
fi
echo
echo "Security audit reports written to: $REPORTS_DIR"
if [[ "$finding_status" -ne 0 && "$FAIL_ON_FINDINGS" != "1" ]]; then
echo "Findings were reported, but this run is report-only. Re-run with --strict to fail on findings."
fi
if [[ "$missing_status" -ne 0 ]]; then
echo "One or more required tools were missing." >&2
fi
exit "$overall_status"

239
tools/checks/check_dependency_boundaries.py Normal file
View File

@@ -0,0 +1,239 @@
#!/usr/bin/env python3
from __future__ import annotations
import ast
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
META_ROOT = Path(__file__).resolve().parents[2]
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
REPOS_ROOT = Path(os.environ.get("GOVOPLAN_REPOS_ROOT", CORE_ROOT.parent)).resolve()
REPOS = {
"core": CORE_ROOT / "src" / "govoplan_core",
"access": REPOS_ROOT / "govoplan-access" / "src" / "govoplan_access",
"admin": REPOS_ROOT / "govoplan-admin" / "src" / "govoplan_admin",
"tenancy": REPOS_ROOT / "govoplan-tenancy" / "src" / "govoplan_tenancy",
"organizations": REPOS_ROOT / "govoplan-organizations" / "src" / "govoplan_organizations",
"identity": REPOS_ROOT / "govoplan-identity" / "src" / "govoplan_identity",
"policy": REPOS_ROOT / "govoplan-policy" / "src" / "govoplan_policy",
"audit": REPOS_ROOT / "govoplan-audit" / "src" / "govoplan_audit",
"dashboard": REPOS_ROOT / "govoplan-dashboard" / "src" / "govoplan_dashboard",
"files": REPOS_ROOT / "govoplan-files" / "src" / "govoplan_files",
"mail": REPOS_ROOT / "govoplan-mail" / "src" / "govoplan_mail",
"campaign": REPOS_ROOT / "govoplan-campaign" / "src" / "govoplan_campaign",
}
PREFIXES = {
"core": "govoplan_core",
"access": "govoplan_access",
"admin": "govoplan_admin",
"tenancy": "govoplan_tenancy",
"organizations": "govoplan_organizations",
"identity": "govoplan_identity",
"policy": "govoplan_policy",
"audit": "govoplan_audit",
"dashboard": "govoplan_dashboard",
"files": "govoplan_files",
"mail": "govoplan_mail",
"campaign": "govoplan_campaign",
}
FEATURE_OWNERS = ("files", "mail", "campaign")
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
WEBUI_REPOS = {
"core": CORE_ROOT / "webui",
"files": REPOS_ROOT / "govoplan-files" / "webui",
"mail": REPOS_ROOT / "govoplan-mail" / "webui",
"campaign": REPOS_ROOT / "govoplan-campaign" / "webui",
}
WEBUI_PACKAGES = {
"core": "@govoplan/core-webui",
"files": "@govoplan/files-webui",
"mail": "@govoplan/mail-webui",
"campaign": "@govoplan/campaign-webui",
}
WEBUI_IMPORT_RE = re.compile(
r"(?:from\s+|import\s*\(\s*|require\s*\(\s*|export\s+[^;]*?\s+from\s+)"
r"[\"'](?P<package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
r"|import\s+[\"'](?P<side_effect_package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
)
@dataclass(frozen=True)
class AllowlistedImport:
owner: str
relative_path: str
imported_owner: str
reason: str
# Transitional exceptions. This should remain empty; add entries only with a
# dated removal plan and a capability/API contract issue.
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
@dataclass(frozen=True)
class Violation:
kind: str
owner: str
path: Path
lineno: int
imported: str
imported_owner: str
def _import_owner(module_name: str) -> str | None:
for owner, prefix in PREFIXES.items():
if module_name == prefix or module_name.startswith(f"{prefix}."):
return owner
return None
def _imports(path: Path) -> list[tuple[int, str]]:
tree = ast.parse(path.read_text(), filename=str(path))
result: list[tuple[int, str]] = []
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
result.append((node.lineno, alias.name))
elif isinstance(node, ast.ImportFrom) and node.module:
result.append((node.lineno, node.module))
return result
def _relative(owner: str, path: Path) -> str:
return path.relative_to(REPOS[owner]).as_posix()
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
def _violations() -> list[Violation]:
violations: list[Violation] = []
for owner, root in REPOS.items():
if not root.exists():
continue
for path in root.rglob("*.py"):
if "__pycache__" in path.parts:
continue
for lineno, imported in _imports(path):
imported_owner = _import_owner(imported)
if imported_owner is None or imported_owner == owner:
continue
if owner == "core" and imported_owner in FEATURE_OWNERS:
if not _allowed(owner, path, imported_owner):
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
elif owner == "access" and imported_owner in FEATURE_OWNERS:
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
elif owner in PLATFORM_OWNERS and imported_owner == "access":
if not _allowed(owner, path, imported_owner):
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
if not _allowed(owner, path, imported_owner):
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
elif owner in FEATURE_OWNERS and imported_owner == "access":
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
violations.extend(_webui_violations())
return violations
def _webui_package_owner(package_name: str) -> str | None:
for owner, package in WEBUI_PACKAGES.items():
if package_name == package:
return owner
return None
def _webui_source_files(root: Path) -> list[Path]:
source_root = root / "src"
if not source_root.exists():
return []
return [
path
for path in source_root.rglob("*")
if path.suffix in {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
and "node_modules" not in path.parts
and "dist" not in path.parts
]
def _webui_dependency_fields(payload: object) -> dict[str, object]:
if not isinstance(payload, dict):
return {}
result: dict[str, object] = {}
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
value = payload.get(key)
if isinstance(value, dict):
result.update(value)
return result
def _webui_package_json_violations(owner: str, root: Path) -> list[Violation]:
violations: list[Violation] = []
package_paths = [root / "package.json"]
if root.name == "webui":
package_paths.append(root.parent / "package.json")
for package_path in tuple(dict.fromkeys(package_paths)):
if not package_path.exists():
continue
try:
payload = json.loads(package_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return [Violation("webui-package", owner, package_path, exc.lineno, "invalid package.json", owner)]
for package_name in _webui_dependency_fields(payload):
imported_owner = _webui_package_owner(str(package_name))
if imported_owner is None or imported_owner == owner:
continue
if owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
violations.append(Violation("webui-package", owner, package_path, 1, str(package_name), imported_owner))
return violations
def _webui_source_violations(owner: str, root: Path) -> list[Violation]:
violations: list[Violation] = []
for path in _webui_source_files(root):
text = path.read_text(encoding="utf-8")
for match in WEBUI_IMPORT_RE.finditer(text):
package_name = match.group("package") or match.group("side_effect_package")
imported_owner = _webui_package_owner(package_name)
if imported_owner is None or imported_owner == owner:
continue
lineno = text.count("\n", 0, match.start()) + 1
if owner == "core" and imported_owner in FEATURE_OWNERS:
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
return violations
def _webui_violations() -> list[Violation]:
violations: list[Violation] = []
for owner, root in WEBUI_REPOS.items():
if not root.exists():
continue
violations.extend(_webui_package_json_violations(owner, root))
violations.extend(_webui_source_violations(owner, root))
return violations
def main() -> int:
violations = _violations()
if not violations:
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
return 0
print("Dependency boundary violations:")
for item in violations:
print(
f"- {item.path}:{item.lineno}: {item.owner} {item.kind} imports {item.imported} "
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
)
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
return 1
if __name__ == "__main__":
raise SystemExit(main())

513
tools/checks/module-installer-rollback-drill.py Normal file
View File

@@ -0,0 +1,513 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from urllib.error import URLError
META_ROOT = Path(__file__).resolve().parents[2]
ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
SRC = ROOT / "src"
if str(SRC) not in sys.path:
sys.path.insert(0, str(SRC))
os.environ.setdefault("APP_ENV", "test")
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
os.environ.setdefault("CELERY_ENABLED", "false")
from govoplan_core.admin.models import SystemSettings
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
from govoplan_core.core.module_installer import (
cancel_module_installer_request,
claim_next_module_installer_request,
module_installer_daemon_status,
module_installer_lock_status,
queue_module_installer_request,
read_module_installer_run,
retry_module_installer_request,
rollback_module_install_run,
run_module_install_plan,
supervise_module_install_plan,
update_module_installer_daemon_status,
update_module_installer_request,
)
from govoplan_core.core.module_management import (
ModuleInstallPlan,
ModuleInstallPlanItem,
saved_desired_enabled_modules,
saved_module_install_plan,
save_desired_enabled_modules,
save_module_install_plan,
)
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, get_database
from govoplan_core.server.registry import available_module_manifests
Scenario = Callable[[Path], dict[str, object]]
def main() -> int:
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
args = parser.parse_args()
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
runtime_root.mkdir(parents=True, exist_ok=True)
selected = args.scenario or sorted(SCENARIOS)
results: list[dict[str, object]] = []
ok = True
try:
for name in selected:
scenario_root = runtime_root / name
scenario_root.mkdir(parents=True, exist_ok=True)
try:
result = SCENARIOS[name](scenario_root)
except Exception as exc:
ok = False
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
else:
ok = ok and bool(result.get("ok"))
results.append(result)
finally:
if not args.keep_runtime and args.runtime_root is None:
shutil.rmtree(runtime_root, ignore_errors=True)
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
if args.format == "json":
print(json.dumps(payload, indent=2, sort_keys=True))
else:
for result in results:
status = "ok" if result.get("ok") else "FAILED"
print(f"{status}: {result['scenario']}")
if result.get("run_id"):
print(f" run: {result['run_id']}")
if result.get("record_path"):
print(f" record: {result['record_path']}")
if result.get("error"):
print(f" error: {result['error']}")
print(f"runtime: {runtime_root}")
return 0 if ok else 1
def package_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "package-failure-example")
with database.session() as session, _patch_subprocess(_package_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
)
restored_plan = saved_module_install_plan(session)
record = _run_record(root, result.run_id)
return _scenario_result(
"package-failure",
root,
result,
expected_status="rolled-back",
checks={
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
},
)
def migration_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "migration-failure-example")
with database.session() as session, _patch_subprocess(_migration_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
migrate_database=True,
)
record = _run_record(root, result.run_id)
return _scenario_result(
"migration-failure",
root,
result,
expected_status="rolled-back",
checks={
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
},
)
def restart_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "restart-failure-example")
with database.session() as session, _patch_subprocess(_restart_failure_run):
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
restart_command="restart-fails",
)
record = _run_record(root, result.run_id)
return _scenario_result(
"restart-failure",
root,
result,
expected_status="rolled-back",
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
)
def health_timeout(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
plan = _saved_install_plan(database, "health-timeout-example")
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
result = supervise_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
restart_command="restart-ok",
health_url="http://127.0.0.1:9/health",
health_timeout_seconds=0.2,
health_interval_seconds=0.05,
)
record = _run_record(root, result.run_id)
return _scenario_result(
"health-timeout",
root,
result,
expected_status="rolled-back",
checks={
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
},
)
def destructive_retirement_failure(root: Path) -> dict[str, object]:
database_url, database = _sqlite_database(root)
available = {"retirement-example": _retirement_failure_manifest()}
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": "retirement-example",
"action": "uninstall",
"python_package": "govoplan-retirement-example",
"destroy_data": True,
}])
session.commit()
with _patch_subprocess(_success_run):
result = run_module_install_plan(
session=session,
plan=plan,
available=available,
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url=database_url,
runtime_dir=root / "installer",
)
record = _run_record(root, result.run_id)
return _scenario_result(
"destructive-retirement-failure",
root,
result,
expected_status="rolled-back",
checks={
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
},
)
def postgres_hooks(root: Path) -> dict[str, object]:
_, database = _sqlite_database(root)
backup_script = (
"import json, os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
)
check_script = (
"import os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
)
restore_script = (
"import os, pathlib; "
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
)
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": "postgres-hook-example",
"action": "install",
"python_package": "govoplan-postgres-hook-example",
"python_ref": "govoplan-postgres-hook-example==0.1.0",
}])
session.commit()
result = run_module_install_plan(
session=session,
plan=plan,
available=available_module_manifests(),
current_enabled=("tenancy", "access"),
desired_enabled=("tenancy", "access"),
database_url="postgresql://db.example.invalid/govoplan",
runtime_dir=root / "installer",
migrate_database=True,
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
dry_run=True,
)
with _patch_subprocess(_non_shell_success_run):
rollback = rollback_module_install_run(
run_id=result.run_id,
runtime_dir=root / "installer",
database_url="postgresql://db.example.invalid/govoplan",
)
run_dir = result.record_path.parent
record = _run_record(root, result.run_id)
ok = (
result.status == "dry-run"
and rollback.status == "rolled-back"
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
)
return {
"scenario": "postgres-hooks",
"ok": ok,
"root": str(root),
"run_id": result.run_id,
"record_path": str(result.record_path),
"rollback_status": rollback.status,
}
def queue_and_lock(root: Path) -> dict[str, object]:
runtime_dir = root / "installer"
status = update_module_installer_daemon_status(
runtime_dir=runtime_dir,
patch={"status": "polling", "current_request_id": None},
)
request = queue_module_installer_request(
runtime_dir=runtime_dir,
requested_by="drill",
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
)
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
assert claimed is not None
failed = update_module_installer_request(
runtime_dir=runtime_dir,
request_id=str(claimed["request_id"]),
patch={"status": "failed", "error": "simulated drill failure"},
)
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
lock_path = runtime_dir / "install.lock"
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
lock = module_installer_lock_status(runtime_dir=runtime_dir)
lock_path.unlink()
return {
"scenario": "queue-and-lock",
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
"root": str(root),
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
}
def _sqlite_database(root: Path) -> tuple[str, Any]:
database_url = f"sqlite:///{root / 'drill.db'}"
configure_database(database_url)
database = get_database()
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
return database_url, database
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
with database.session() as session:
save_maintenance_mode(session, MaintenanceMode(enabled=True))
plan = save_module_install_plan(session, [{
"module_id": module_id,
"action": "install",
"python_package": f"govoplan-{module_id}",
"python_ref": f"govoplan-{module_id}==0.1.0",
}])
save_desired_enabled_modules(session, ("tenancy", "access"))
session.commit()
return plan
def _retirement_failure_manifest() -> ModuleManifest:
def destroy(_session: object, _module_id: str) -> None:
raise RuntimeError("simulated destructive retirement failure")
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
return MigrationRetirementPlan(
supported=True,
summary="Drill retirement provider supports destructive cleanup.",
destroy_data_supported=True,
destroy_data_summary="Drill destructive cleanup will fail during execution.",
destroy_data_executor=destroy,
)
return ModuleManifest(
id="retirement-example",
name="Retirement example",
version="0.1.0",
migration_spec=MigrationSpec(
module_id="retirement-example",
retirement_supported=True,
retirement_provider=provider,
),
)
def _scenario_result(
scenario: str,
root: Path,
result: Any,
*,
expected_status: str,
checks: dict[str, bool],
) -> dict[str, object]:
return {
"scenario": scenario,
"ok": result.status == expected_status and all(checks.values()),
"root": str(root),
"run_id": result.run_id,
"record_path": str(result.record_path),
"status": result.status,
"checks": checks,
}
def _run_record(root: Path, run_id: str) -> dict[str, object]:
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
def _nested(value: dict[str, object], *keys: str) -> object:
current: object = value
for key in keys:
if not isinstance(current, dict):
return None
current = current.get(key)
return current
def _command_text(argv: object) -> str:
if isinstance(argv, list | tuple):
return " ".join(str(item) for item in argv)
return str(argv)
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "pip freeze" in text:
return _completed(stdout="govoplan-core==0.0.0\n")
return _completed()
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
if kwargs.get("shell"):
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
return _success_run(argv, *_args, **kwargs)
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "pip install" in text and "==" in text and "-r" not in text:
return _completed(23, stderr="simulated package install failure")
return _success_run(argv, *_args, **kwargs)
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
text = _command_text(argv)
if "govoplan_core.commands.init_db" in text:
return _completed(24, stderr="simulated migration failure")
return _success_run(argv, *_args, **kwargs)
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
return _completed(25, stderr="simulated restart failure")
return _success_run(argv, *_args, **kwargs)
ORIGINAL_SUBPROCESS_RUN = subprocess.run
@contextmanager
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
import govoplan_core.core.module_installer as installer
original = installer.subprocess.run
installer.subprocess.run = fake_run # type: ignore[assignment]
try:
yield
finally:
installer.subprocess.run = original # type: ignore[assignment]
@contextmanager
def _patch_urlopen() -> Iterable[None]:
import govoplan_core.core.module_installer as installer
original = installer.urllib.request.urlopen
def fail(*_args: object, **_kwargs: object) -> None:
raise URLError("simulated health timeout")
installer.urllib.request.urlopen = fail # type: ignore[assignment]
try:
yield
finally:
installer.urllib.request.urlopen = original # type: ignore[assignment]
SCENARIOS: dict[str, Scenario] = {
"destructive-retirement-failure": destructive_retirement_failure,
"health-timeout": health_timeout,
"migration-failure": migration_failure,
"package-failure": package_failure,
"postgres-hooks": postgres_hooks,
"queue-and-lock": queue_and_lock,
"restart-failure": restart_failure,
}
if __name__ == "__main__":
raise SystemExit(main())

129
tools/checks/postgres-integration-check.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import base64
import os
from pathlib import Path
import subprocess
import sys
META_ROOT = Path(__file__).resolve().parents[2]
ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
SRC = ROOT / "src"
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
("core-only", "tenancy,access,admin"),
("files-only", "tenancy,access,admin,files"),
("mail-only", "tenancy,access,admin,mail"),
("campaign-only", "tenancy,access,admin,campaigns"),
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
)
def main() -> int:
parser = argparse.ArgumentParser(
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
)
parser.add_argument(
"--database-url",
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
)
parser.add_argument(
"--module-set",
action="append",
default=[],
metavar="NAME=MODULES",
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
)
parser.add_argument(
"--reset-schema",
action="store_true",
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
)
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
args = parser.parse_args()
database_url = (args.database_url or "").strip()
if not database_url:
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
if not database_url.startswith("postgresql"):
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
module_sets = _parse_module_sets(args.module_set)
ok = True
for name, modules in module_sets:
print(f"\n== {name}: {modules} ==")
if args.reset_schema:
_reset_public_schema(database_url)
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
if not args.skip_migrate:
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
if not args.skip_smoke:
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
return 0 if ok else 1
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
if not raw:
return DEFAULT_MODULE_SETS
parsed: list[tuple[str, str]] = []
for item in raw:
name, separator, modules = item.partition("=")
if not separator or not name.strip() or not modules.strip():
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
parsed.append((name.strip(), modules.strip()))
return tuple(parsed)
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
env = os.environ.copy()
env["APP_ENV"] = app_env
env["DATABASE_URL"] = database_url
env["ENABLED_MODULES"] = modules
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
env["DEV_BOOTSTRAP_ENABLED"] = "false"
env["CELERY_ENABLED"] = "false"
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
env.setdefault("FILE_STORAGE_BACKEND", "local")
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
return env
def _run(command: list[str], *, env: dict[str, str]) -> bool:
print("+ " + " ".join(command))
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
if result.returncode != 0:
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
return False
return True
def _reset_public_schema(database_url: str) -> None:
try:
from sqlalchemy import create_engine
except ModuleNotFoundError as exc:
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
try:
with engine.connect() as connection:
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
connection.exec_driver_sql("CREATE SCHEMA public")
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
finally:
engine.dispose()
print("Reset PostgreSQL schema: public")
if __name__ == "__main__":
raise SystemExit(main())

54
tools/checks/security-audit/Dockerfile Normal file
View File

@@ -0,0 +1,54 @@
FROM golang:1.26-bookworm AS go-tools
ARG GITLEAKS_VERSION=v8.30.1
ARG OSV_SCANNER_VERSION=v2.4.0
RUN go install github.com/zricethezav/gitleaks/v8@${GITLEAKS_VERSION} \
&& go install github.com/google/osv-scanner/v2/cmd/osv-scanner@${OSV_SCANNER_VERSION}
FROM python:3.12-bookworm
ENV DEBIAN_FRONTEND=noninteractive \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PYTHONDONTWRITEBYTECODE=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
git \
gnupg \
jq \
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& curl -fsSL https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor -o /usr/share/keyrings/trivy.gpg \
&& printf 'deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main\n' > /etc/apt/sources.list.d/trivy.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends trivy \
&& rm -rf /var/lib/apt/lists/*
COPY --from=go-tools /go/bin/gitleaks /usr/local/bin/gitleaks
COPY --from=go-tools /go/bin/osv-scanner /usr/local/bin/osv-scanner
COPY requirements-audit.txt /tmp/requirements-audit.txt
RUN python -m pip install --upgrade pip \
&& python -m pip install --no-cache-dir -r /tmp/requirements-audit.txt \
&& npm install -g jscpd \
&& semgrep --version \
&& bandit --version \
&& ruff --version \
&& gitleaks version \
&& osv-scanner --version \
&& trivy --version \
&& jscpd --version
RUN mkdir -p /workspace \
&& chmod 0777 /workspace
WORKDIR /workspace
USER 65532:65532
HEALTHCHECK NONE
CMD ["bash", "tools/checks/check-security-audit.sh", "--mode", "ci", "--scope", "current"]

136
tools/checks/security-audit/run.sh Normal file
View File

@@ -0,0 +1,136 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
IMAGE="${SECURITY_AUDIT_IMAGE:-govoplan/security-audit:local}"
SCOPE="${SECURITY_AUDIT_SCOPE:-current}"
MODE="${SECURITY_AUDIT_MODE:-full}"
REPORTS_DIR="${SECURITY_AUDIT_REPORTS_DIR:-audit-reports}"
REBUILD="${SECURITY_AUDIT_REBUILD:-0}"
UPDATE="${SECURITY_AUDIT_UPDATE:-0}"
BUILD_ONLY=0
SCRIPT_ARGS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--mode)
MODE="${2:?missing mode}"
shift 2
;;
--scope)
SCOPE="${2:?missing scope}"
shift 2
;;
--reports-dir)
REPORTS_DIR="${2:?missing reports dir}"
shift 2
;;
--rebuild)
REBUILD=1
shift
;;
--update)
UPDATE=1
REBUILD=1
shift
;;
--build-only)
BUILD_ONLY=1
shift
;;
-h|--help)
cat <<'EOF'
Usage: tools/checks/security-audit/run.sh [--mode quick|ci|full] [--scope current|govoplan] [--reports-dir DIR] [--rebuild] [--update] [--build-only] [--strict|--report-only]
Builds or reuses the containerized GovOPlaN audit toolbox, then runs
tools/checks/check-security-audit.sh inside it.
Image caching:
The image is tagged by a fingerprint of tools/checks/security-audit/Dockerfile and
requirements-audit.txt. Existing matching images are reused.
Flags:
--rebuild Rebuild the fingerprinted image using Docker cache.
--update Rebuild with --pull --no-cache to refresh base images/tools.
--build-only Build or refresh the image, then exit without running an audit.
EOF
exit 0
;;
--strict|--report-only)
SCRIPT_ARGS+=("$1")
shift
;;
*)
SCRIPT_ARGS+=("$1")
shift
;;
esac
done
REBUILD="${REBUILD:-0}"
UPDATE="${UPDATE:-0}"
BUILD_ONLY="${BUILD_ONLY:-0}"
IMAGE_NAME="${IMAGE##*/}"
if [[ "$IMAGE_NAME" == *:* ]]; then
IMAGE_REPOSITORY="${IMAGE%:*}"
IMAGE_ALIAS="$IMAGE"
else
IMAGE_REPOSITORY="$IMAGE"
IMAGE_ALIAS="$IMAGE:local"
fi
IMAGE_FINGERPRINT="$(
{
sha256sum "$ROOT/tools/checks/security-audit/Dockerfile"
sha256sum "$ROOT/requirements-audit.txt"
} | sha256sum | cut -c1-16
)"
FINGERPRINT_IMAGE="${SECURITY_AUDIT_FINGERPRINT_IMAGE:-$IMAGE_REPOSITORY:$IMAGE_FINGERPRINT}"
build_image() {
local -a build_args=(-t "$FINGERPRINT_IMAGE" -t "$IMAGE_ALIAS" -f "$ROOT/tools/checks/security-audit/Dockerfile")
if [[ "$UPDATE" == "1" ]]; then
build_args=(--pull --no-cache "${build_args[@]}")
fi
docker build "${build_args[@]}" "$ROOT"
}
if [[ "$REBUILD" == "1" ]] || ! docker image inspect "$FINGERPRINT_IMAGE" >/dev/null 2>&1; then
echo "Building security audit toolbox image: $FINGERPRINT_IMAGE"
build_image
else
echo "Reusing security audit toolbox image: $FINGERPRINT_IMAGE"
if ! docker image inspect "$IMAGE_ALIAS" >/dev/null 2>&1; then
docker tag "$FINGERPRINT_IMAGE" "$IMAGE_ALIAS"
fi
fi
if [[ "$BUILD_ONLY" == "1" ]]; then
docker image inspect "$FINGERPRINT_IMAGE" --format 'Built audit toolbox image {{.RepoTags}} {{.Id}}'
exit 0
fi
if [[ "$SCOPE" == "govoplan" ]]; then
MOUNT_ROOT="$(dirname "$ROOT")"
WORKDIR="/workspace/$(basename "$ROOT")"
EXTRA_ENV=(-e "GOVOPLAN_REPOS_ROOT=/workspace")
else
MOUNT_ROOT="$ROOT"
WORKDIR="/workspace"
EXTRA_ENV=()
fi
docker run --rm \
--user "$(id -u):$(id -g)" \
-e HOME=/tmp/security-audit-home \
-e SECURITY_AUDIT_SCOPE="$SCOPE" \
-e SECURITY_AUDIT_MODE="$MODE" \
-e SECURITY_AUDIT_REPORTS_DIR="$REPORTS_DIR" \
-e SECURITY_AUDIT_FAIL_ON_FINDINGS="${SECURITY_AUDIT_FAIL_ON_FINDINGS:-0}" \
-e SECURITY_AUDIT_REQUIRE_TOOLS="${SECURITY_AUDIT_REQUIRE_TOOLS:-0}" \
"${EXTRA_ENV[@]}" \
-v "$MOUNT_ROOT:/workspace" \
-w "$WORKDIR" \
"$FINGERPRINT_IMAGE" \
bash tools/checks/check-security-audit.sh --mode "$MODE" --scope "$SCOPE" --reports-dir "$REPORTS_DIR" "${SCRIPT_ARGS[@]}"

51
tools/checks/security-audit/semgrep-govoplan.yml Normal file
View File

@@ -0,0 +1,51 @@
rules:
- id: govoplan.python.subprocess-shell-true
languages: [python]
severity: WARNING
message: Avoid shell=True for module/runtime commands; prefer argv lists and explicit argument validation.
patterns:
- pattern-either:
- pattern: subprocess.run(..., shell=True, ...)
- pattern: subprocess.call(..., shell=True, ...)
- pattern: subprocess.check_call(..., shell=True, ...)
- pattern: subprocess.check_output(..., shell=True, ...)
- id: govoplan.python.os-system
languages: [python]
severity: WARNING
message: Avoid os.system; use subprocess with argv lists and explicit environment handling.
pattern: os.system(...)
- id: govoplan.python.pickle-load
languages: [python]
severity: WARNING
message: Pickle loading is unsafe for untrusted data; use structured formats or signed payloads.
pattern-either:
- pattern: pickle.load(...)
- pattern: pickle.loads(...)
- id: govoplan.python.cors-wildcard-credentials
languages: [python]
severity: ERROR
message: Credentialed CORS must not use wildcard origins.
pattern: CORSMiddleware(..., allow_origins=["*"], allow_credentials=True, ...)
- id: govoplan.web.localstorage-sensitive-value
languages: [javascript, typescript]
severity: WARNING
message: Do not persist tokens, API keys, credentials, or passwords in localStorage.
patterns:
- pattern: localStorage.setItem($KEY, $VALUE)
- metavariable-regex:
metavariable: $KEY
regex: (?i).*(token|api.?key|secret|password|credential).*
- id: govoplan.web.window-localstorage-sensitive-value
languages: [javascript, typescript]
severity: WARNING
message: Do not persist tokens, API keys, credentials, or passwords in localStorage.
patterns:
- pattern: window.localStorage.setItem($KEY, $VALUE)
- metavariable-regex:
metavariable: $KEY
regex: (?i).*(token|api.?key|secret|password|credential).*

580
tools/gitea/gitea-backlog-import.py Normal file
View File

@@ -0,0 +1,580 @@
#!/usr/bin/env python3
"""Import markdown backlog/planning files into Gitea issues."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import os
import pathlib
import re
import sys
from typing import Any, Iterable
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/addideas-govoplan-website/TODO.md")
REPO_ROOTS = {
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
"web": pathlib.Path("/mnt/DATA/git/addideas-govoplan-website"),
}
MODULE_LABELS = {
"core": "module/core",
"access": "module/access",
"mail": "module/mail",
"files": "module/files",
"campaign": "module/campaign",
"web": "area/marketing",
}
@dataclasses.dataclass(frozen=True)
class Candidate:
repo_key: str
title: str
body: str
labels: tuple[str, ...]
fingerprint: str
source: str
line: int
@dataclasses.dataclass
class RepoState:
target: Any
client: GiteaClient
label_ids: dict[str, int]
fingerprints: set[str]
titles: set[str]
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
args = parser.parse_args()
try:
load_dotenv(args.env_file)
candidates = list(build_candidates(args))
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
for repo_key, grouped in group_candidates(candidates).items():
print(f"{repo_key}: {len(grouped)} candidate(s)")
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
if not token:
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
print_preview(candidates)
return 0
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
skipped = len(candidates) - len(missing)
print(f"Skipped {skipped} existing issue candidate(s).")
print(f"Missing {len(missing)} issue candidate(s).")
print_preview(missing)
if not args.apply:
print("Dry run only. Re-run with --apply to create missing issues.")
return 0
created_by_repo: dict[str, list[str]] = {}
for candidate in missing:
state = states[candidate.repo_key]
validate_labels(candidate, state)
issue = state.client.request_json(
"POST",
repo_path(state.target.owner, state.target.repo, "/issues"),
body={
"title": candidate.title,
"body": candidate.body,
"labels": [state.label_ids[label] for label in candidate.labels],
},
)
number = issue.get("number") or issue.get("index")
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
state.fingerprints.add(candidate.fingerprint)
state.titles.add(normalize_title(candidate.title))
print("Created issues:")
for repo_key, titles in created_by_repo.items():
print(f"{repo_key}:")
for title in titles:
print(f" {title}")
if not created_by_repo:
print(" none")
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
if args.product_backlog.exists():
yield from parse_product_backlog(args.product_backlog)
if args.access_plan and args.access_plan.exists():
yield from parse_access_plan(args.access_plan)
if args.web_todo.exists():
yield from parse_simple_todo(args.web_todo)
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
headings: list[tuple[int, str]] = []
status = ""
lines = path.read_text(encoding="utf-8").splitlines()
for index, line in enumerate(lines):
line_number = index + 1
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
if heading_match:
level = len(heading_match.group(1))
title = heading_match.group(2).strip()
if level == 2 and title == "Completed Work Archive":
break
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
headings.append((level, title))
status = ""
continue
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
if status_match:
status = status_match.group(1).strip()
continue
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
if not item_match:
continue
item = collect_continued_item(lines, index, item_match.group(1))
section = section_title(headings)
repo_key, labels = classify_product_item(item, section)
prefix = type_prefix(labels)
title = format_title(prefix, item)
source = str(path)
body = body_for_item(
fingerprint=make_fingerprint(source, line_number, repo_key, item),
source=source,
line=line_number,
section=section,
status=status,
item=item,
note="Imported from the consolidated GovOPlaN product backlog.",
)
yield Candidate(
repo_key=repo_key,
title=title,
body=body,
labels=tuple(sorted(set(labels))),
fingerprint=make_fingerprint(source, line_number, repo_key, item),
source=source,
line=line_number,
)
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
yield from access_stage_candidates(path, lines)
yield from access_immediate_candidates(path, lines)
yield from access_decision_candidates(path, lines)
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
index = 0
while index < len(lines):
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
if not match:
index += 1
continue
stage = match.group(1)
name = match.group(2).strip()
start_line = index + 1
index += 1
block: list[str] = []
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
block.append(lines[index])
index += 1
item = f"Access extraction Stage {stage}: {name}"
fingerprint = make_fingerprint(str(path), start_line, "core", item)
body = "\n".join(
[
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
"",
"Imported from the GovOPlaN access extraction plan.",
"",
f"- Source: `{path}`",
f"- Line: `{start_line}`",
f"- Section: `Stage {stage}: {name}`",
"",
"Plan excerpt:",
"",
"```markdown",
f"### Stage {stage}: {name}",
*block,
"```",
]
)
yield Candidate(
repo_key="core",
title=format_title("[Task]", item),
body=body,
labels=labels("core", "type/task", "area/module-system", "module/access"),
fingerprint=fingerprint,
source=str(path),
line=start_line,
)
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
in_section = False
for index, line in enumerate(lines):
line_number = index + 1
if line == "## Immediate Backlog":
in_section = True
continue
if in_section and line.startswith("## "):
break
if not in_section:
continue
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
if not item_match:
continue
item = collect_continued_item(lines, index, item_match.group(1))
title_item = access_title_item(item)
fingerprint = make_fingerprint(str(path), line_number, "core", item)
body = body_for_item(
fingerprint=fingerprint,
source=str(path),
line=line_number,
section="Immediate Backlog",
status="open",
item=item,
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
)
yield Candidate(
repo_key="core",
title=format_title("[Task]", title_item),
body=body,
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
fingerprint=fingerprint,
source=str(path),
line=line_number,
)
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
in_section = False
for index, line in enumerate(lines):
line_number = index + 1
if line == "## Open Decisions":
in_section = True
continue
if in_section and line.startswith("## "):
break
if not in_section:
continue
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
if not item_match:
continue
item = collect_continued_item(lines, index, item_match.group(1))
fingerprint = make_fingerprint(str(path), line_number, "core", item)
body = body_for_item(
fingerprint=fingerprint,
source=str(path),
line=line_number,
section="Open Decisions",
status="decision needed",
item=item,
note="Imported from the GovOPlaN access extraction plan open decisions.",
)
yield Candidate(
repo_key="core",
title=format_title("[Task]", f"Decide: {item}"),
body=body,
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
fingerprint=fingerprint,
source=str(path),
line=line_number,
)
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
lines = path.read_text(encoding="utf-8").splitlines()
for index, line in enumerate(lines):
line_number = index + 1
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
if not item_match:
continue
item = collect_continued_item(lines, index, item_match.group(1))
fingerprint = make_fingerprint(str(path), line_number, "web", item)
yield Candidate(
repo_key="web",
title=format_title("[Task]", item),
body=body_for_item(
fingerprint=fingerprint,
source=str(path),
line=line_number,
section="TODO",
status="open",
item=item,
note="Imported from the GovOPlaN website TODO file.",
),
labels=labels("web", "type/task", "area/marketing"),
fingerprint=fingerprint,
source=str(path),
line=line_number,
)
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
text = f"{section} {item}".lower()
repo_key = "core"
area = "area/devex"
issue_type = "type/task"
if "dsar search/export/delete" in text:
return "core", labels("core", "type/feature", "area/governance")
if "fully stream zip uploads" in text:
return "files", labels("files", "type/task", "area/api")
if "installer packaging approach" in text:
return "core", labels("core", "type/task", "area/devex")
if "profile reselection/revalidation" in text:
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
if "module boundaries" in text:
return "core", labels("core", issue_type, "area/module-system")
if "external storage connectors" in text:
repo_key = "files"
area = "area/api"
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
repo_key = "campaign"
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
elif "controlled sending" in text:
repo_key = "campaign"
area = "area/api"
elif "mail profiles" in text:
repo_key = "mail"
area = "area/api"
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
repo_key = "mail"
area = "area/api"
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
repo_key = "files"
area = "area/api"
if "external storage connectors" not in text and any(
word in text
for word in (
"campaign",
"recipient",
"attachment",
"zip",
"evidence",
"wizard",
"send",
"archive",
"message preview",
"address book",
"carddav",
"mailing lists",
)
):
repo_key = "campaign"
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
repo_key = "core"
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
repo_key = "core"
area = "area/module-system"
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
repo_key = "core"
area = "area/release"
if repo_key == "core" and not any(
phrase in text
for phrase in (
"external storage connectors",
"controlled sending",
"report, evidence",
"recipient import",
"campaign and attachment",
)
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
repo_key = "core"
area = "area/devex"
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
area = "area/docs"
if "ui lists often start at index 2" in text:
issue_type = "type/bug"
area = "area/webui"
repo_key = "core"
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
issue_type = "type/feature"
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
issue_type = "type/debt"
return repo_key, labels(repo_key, issue_type, area)
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
parts = [initial.strip()]
next_index = index + 1
while next_index < len(lines):
line = lines[next_index]
if not line.startswith((" ", "\t")):
break
stripped = line.strip()
if not stripped:
break
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
break
parts.append(stripped)
next_index += 1
return " ".join(parts)
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
base.update(extra)
if "status/needs-info" in base:
base.discard("status/triage")
return tuple(sorted(base))
def type_prefix(label_set: Iterable[str]) -> str:
labels_set = set(label_set)
if "type/bug" in labels_set:
return "[Bug]"
if "type/feature" in labels_set:
return "[Feature]"
if "type/debt" in labels_set:
return "[Debt]"
if "type/docs" in labels_set:
return "[Docs]"
return "[Task]"
def format_title(prefix: str, item: str) -> str:
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
title = f"{prefix} {clean}"
if len(title) <= 180:
return title
return f"{title[:177].rstrip()}..."
def access_title_item(item: str) -> str:
lowered = item.lower()
if "compatibility wrappers" in lowered:
return "Wire core auth compatibility through access capabilities"
return item
def body_for_item(
*,
fingerprint: str,
source: str,
line: int,
section: str,
status: str,
item: str,
note: str,
) -> str:
lines = [
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
"",
note,
"",
f"- Source: `{source}`",
f"- Line: `{line}`",
f"- Section: `{section}`",
]
if status:
lines.append(f"- Source status: `{status}`")
lines.extend(
[
"",
"Imported backlog item:",
"",
"```markdown",
f"- [ ] {item}",
"```",
]
)
return "\n".join(lines)
def section_title(headings: list[tuple[int, str]]) -> str:
return " > ".join(title for _, title in headings)
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
stable = "\n".join([source, str(line), repo_key, item])
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
grouped: dict[str, list[Candidate]] = {}
for candidate in candidates:
grouped.setdefault(candidate.repo_key, []).append(candidate)
return grouped
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
states: dict[str, RepoState] = {}
for repo_key in repo_keys:
root = repo_root(REPO_ROOTS[repo_key])
target = infer_target(root)
client = GiteaClient(target, token)
label_ids = label_ids_by_name(client, target.owner, target.repo)
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
fingerprints: set[str] = set()
titles: set[str] = set()
for issue in issues:
body = str(issue.get("body") or "")
if issue.get("state") == "closed" and "Moved to `" in body:
continue
titles.add(normalize_title(str(issue.get("title") or "")))
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
return states
def already_imported(candidate: Candidate, state: RepoState) -> bool:
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
def normalize_title(title: str) -> str:
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
def validate_labels(candidate: Candidate, state: RepoState) -> None:
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
if missing:
joined = ", ".join(missing)
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
for repo_key, grouped in group_candidates(candidates).items():
print(f"{repo_key}:")
for candidate in grouped[:limit]:
print(f" {candidate.title}")
if len(grouped) > limit:
print(f" ... {len(grouped) - limit} more")
if __name__ == "__main__":
raise SystemExit(main())

97
tools/gitea/gitea-codex-note.py Normal file
View File

@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Post a standardized Codex state update comment to a Gitea issue."""
from __future__ import annotations
import argparse
import os
import pathlib
import sys
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
STATUS_LABELS = {
"started": "status/in-progress",
"progress": "status/in-progress",
"blocked": "status/blocked",
"needs-info": "status/needs-info",
"ready": "status/ready",
"done": "",
"note": "",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
args = parser.parse_args()
try:
root = repo_root(args.root)
load_dotenv(args.env_file or root / ".env")
target = infer_target(root, args.remote)
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
body = build_body(args)
print(f"Target: {target.display}")
print(f"Issue: #{args.issue}")
print(body)
if not args.apply:
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
return 0
client = GiteaClient(target, token)
client.request_json(
"POST",
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
body={"body": body},
)
if args.close:
client.request_json(
"PATCH",
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
body={"state": "closed"},
)
print("Posted Codex note.")
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
def build_body(args: argparse.Namespace) -> str:
lines = [f"## Codex State: {args.status}", ""]
append_section(lines, "Summary", args.summary)
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
append_section(lines, "Next / Blocked", args.next)
if STATUS_LABELS[args.status]:
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
if args.body_file:
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
return "\n".join(lines).rstrip() + "\n"
def append_section(lines: list[str], title: str, items: list[str]) -> None:
if not items:
return
lines.extend([f"### {title}", ""])
lines.extend(f"- {item}" for item in items)
lines.append("")
if __name__ == "__main__":
raise SystemExit(main())

600
tools/gitea/gitea-import-all-backlogs.py Normal file
View File

@@ -0,0 +1,600 @@
#!/usr/bin/env python3
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
from __future__ import annotations
import argparse
import csv
import dataclasses
import hashlib
import os
import pathlib
import re
import subprocess
import sys
from typing import Any, Iterable
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
EXCLUDED_FILE_PATTERNS = (
"/.git/",
"/.venv/",
"/venv/",
"/site-packages/",
"/node_modules/",
"/__pycache__/",
"/dist/",
"/build/",
"/.cache/",
"/.module-test-build/",
)
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
ACTION_RE = re.compile(
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
re.IGNORECASE,
)
SKIP_SECTION_RE = re.compile(
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
r"available now|done|archive)",
re.IGNORECASE,
)
ACTIVE_SECTION_RE = re.compile(
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
re.IGNORECASE,
)
GENERIC_LABELS = [
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
("priority/p0", "b60205", "Immediate stop-the-line priority."),
("priority/p1", "d93f0b", "High priority for the next focused work window."),
("priority/p2", "fbca04", "Normal planned priority."),
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
]
@dataclasses.dataclass(frozen=True)
class RepoInfo:
root: pathlib.Path
remote: str
owner: str
repo: str
@property
def key(self) -> str:
return self.root.name
@dataclasses.dataclass(frozen=True)
class SourceFile:
repo_key: str
path: pathlib.Path
source_kind: str
@dataclasses.dataclass(frozen=True)
class Candidate:
repo_key: str
title: str
body: str
labels: tuple[str, ...]
priority: str
milestone: str
fingerprint: str
source: str
line: int
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
try:
load_dotenv(args.env_file)
repos = discover_hosted_repos(args.git_root)
sources = discover_sources(repos, args.products_root)
candidates = list(build_candidates(sources))
candidates = dedupe_candidates(candidates)
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
print(f"Hosted repositories: {len(repos)}")
print(f"Backlog-like source files: {len(sources)}")
print(f"Issue candidates: {len(candidates)}")
for repo_key, count in grouped_counts(candidates).items():
print(f" {repo_key}: {count}")
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
if not token:
print_preview(candidates)
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
return 0
missing_by_repo: dict[str, list[Candidate]] = {}
skipped = 0
states: dict[str, RepoState] = {}
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
for candidate in candidates:
state = states[candidate.repo_key]
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
skipped += 1
continue
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
missing_total = sum(len(items) for items in missing_by_repo.values())
print(f"Skipped existing: {skipped}")
print(f"Missing candidates: {missing_total}")
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
if not args.apply:
print("Dry run only. Re-run with --apply to create missing issues.")
return 0
for repo_key, repo_candidates in missing_by_repo.items():
state = states[repo_key]
for candidate in repo_candidates:
label_ids = [state.label_ids[label] for label in candidate.labels]
if candidate.priority not in candidate.labels:
label_ids.append(state.label_ids[candidate.priority])
milestone_id = ensure_milestone(state, candidate.milestone)
created = state.client.request_json(
"POST",
repo_path(state.target.owner, state.target.repo, "/issues"),
body={
"title": candidate.title,
"body": candidate.body,
"labels": label_ids,
"milestone": milestone_id,
},
)
number = created.get("number") or created.get("index")
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
state.fingerprints.add(candidate.fingerprint)
state.titles.add(normalize_title(candidate.title))
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
@dataclasses.dataclass
class RepoState:
target: Any
client: GiteaClient
label_ids: dict[str, int]
milestone_ids: dict[str, int]
fingerprints: set[str]
titles: set[str]
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
repos: dict[str, RepoInfo] = {}
for gitdir in sorted(git_root.glob("*/.git")):
root = gitdir.parent
result = subprocess.run(
["git", "-C", str(root), "remote", "get-url", "origin"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
remote = result.stdout.strip()
if "git.add-ideas.de" not in remote:
continue
target = infer_target(root)
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
return repos
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
sources: list[SourceFile] = []
for repo in repos.values():
for path in repo.root.rglob("*"):
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
continue
path_text = path.as_posix()
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
continue
if is_excluded_repo_file(path):
continue
if is_text_backlog(path):
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
product_map = {
"co2api": "emission-api-lib",
"govoplan": "govoplan-core",
"meubility": "meubility-workbench",
"mousehold": "mousehold",
"prvnncr": "prvnncr-server",
}
if products_root.exists():
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
repo_key = product_map.get(product_dir.name)
if not repo_key or repo_key not in repos:
continue
for path in product_dir.rglob("*"):
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
for source in sources:
unique[(source.repo_key, source.path.resolve())] = source
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
def is_excluded_repo_file(path: pathlib.Path) -> bool:
text = path.as_posix()
name = path.name.lower()
if (
"/tools/gitea/gitea-" in text
or text.endswith("/tools/gitea/gitea_common.py")
):
return True
if "testing_plan" in name or "test_plan" in name:
return True
if text.endswith("/docs/GITEA_ISSUES.md"):
return True
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
return True
if text.endswith("/addideas-govoplan-website/TODO.md"):
return True
return False
def is_text_backlog(path: pathlib.Path) -> bool:
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
return False
try:
chunk = path.read_bytes()[:4096]
except OSError:
return False
if b"\x00" in chunk:
return False
return True
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
for source in sources:
suffix = source.path.suffix.lower()
if suffix == ".csv":
yield from parse_csv_source(source)
else:
yield from parse_text_source(source)
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
reader = csv.DictReader(handle)
for index, row in enumerate(reader, start=2):
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
if not story:
continue
identifier = (row.get("id") or "").strip()
priority = normalize_priority(row.get("priority") or "")
milestone = (row.get("epic") or "Backlog").strip()
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
body = "\n".join(
[
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
"",
"Imported from a CSV backlog file.",
"",
f"- Source: `{source.path}`",
f"- Line: `{index}`",
f"- Source kind: `{source.source_kind}`",
f"- Milestone: `{milestone}`",
"",
"CSV row:",
"",
"```text",
", ".join(f"{key}={value}" for key, value in row.items()),
"```",
]
)
yield Candidate(
repo_key=source.repo_key,
title=title,
body=body,
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
priority=priority,
milestone=milestone,
fingerprint=fingerprint,
source=str(source.path),
line=index,
)
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
try:
lines = source.path.read_text(encoding="utf-8").splitlines()
except UnicodeDecodeError:
lines = source.path.read_text(encoding="latin-1").splitlines()
headings: list[tuple[int, str]] = []
in_tasks = False
for index, line in enumerate(lines):
line_number = index + 1
heading = parse_heading(line)
if heading:
level, title = heading
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
headings.append((level, title))
in_tasks = False
continue
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
in_tasks = True
continue
item = parse_open_item(line)
if item is None and is_action_context(source.path, headings, in_tasks):
item = parse_plain_action_item(line)
if item is None:
continue
if not item or should_skip_item(item, headings):
continue
section = section_title(headings)
priority = priority_from_context(section, item)
milestone = milestone_from_context(source.path, section)
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
title = format_title(prefix, item)
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
body = "\n".join(
[
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
"",
"Imported from a backlog-like text file.",
"",
f"- Source: `{source.path}`",
f"- Line: `{line_number}`",
f"- Source kind: `{source.source_kind}`",
f"- Section: `{section or 'none'}`",
"",
"Imported item:",
"",
"```text",
item,
"```",
]
)
yield Candidate(
repo_key=source.repo_key,
title=title,
body=body,
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
priority=priority,
milestone=milestone,
fingerprint=fingerprint,
source=str(source.path),
line=line_number,
)
def parse_heading(line: str) -> tuple[int, str] | None:
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
if markdown:
return len(markdown.group(1)), clean_text(markdown.group(2))
underlined = re.match(r"^(.+?)\s*$", line)
if underlined and line.strip() and len(line.strip()) < 120:
return None
return None
def parse_open_item(line: str) -> str | None:
patterns = [
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
]
for pattern in patterns:
match = re.match(pattern, line)
if match and leading_width(match.group("indent")) == 0:
return clean_text(match.group("text"))
return None
def parse_plain_action_item(line: str) -> str | None:
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
if match and leading_width(match.group("indent")) == 0:
text = clean_text(match.group("text"))
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
return None
if ACTION_RE.match(text) or is_short_noun_task(text):
return text
return None
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
if in_tasks:
return True
context = section_title(headings)
if not context:
return False
if SKIP_SECTION_RE.search(context):
return False
if ACTIVE_SECTION_RE.search(context):
return True
return path.name.lower() in {"todo.txt", "todo.md"}
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
context = section_title(headings)
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
return True
if item.endswith((",", ";")):
return True
if SKIP_SECTION_RE.search(context):
return True
if len(item) < 4:
return True
if item.endswith(":") and len(item.split()) <= 6:
return True
return False
def is_short_noun_task(text: str) -> bool:
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
def clean_text(text: str) -> str:
text = text.strip()
text = re.sub(r"^☐\s*", "", text)
text = re.sub(r"^✅\s*", "", text)
text = re.sub(r"\s+", " ", text)
return text.strip(" -")
def leading_width(indent: str) -> int:
return len(indent.replace("\t", " "))
def section_title(headings: list[tuple[int, str]]) -> str:
return " > ".join(title for _, title in headings)
def priority_from_context(section: str, item: str) -> str:
text = f"{section} {item}".lower()
if "p0" in text or "critical path" in text:
return "priority/p0"
if "p1" in text or "immediate" in text or "next sprint" in text:
return "priority/p1"
if "p2" in text:
return "priority/p2"
if "p3" in text or "future" in text or "later" in text:
return "priority/p3"
return "priority/p2"
def normalize_priority(value: str) -> str:
value = value.strip().lower()
if value == "p0":
return "priority/p0"
if value == "p1":
return "priority/p1"
if value == "p2":
return "priority/p2"
if value == "p3":
return "priority/p3"
return "priority/p2"
def milestone_from_context(path: pathlib.Path, section: str) -> str:
if section:
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
if parts:
return parts[-1][:120]
return path.stem.replace("_", " ").replace("-", " ").title()
def format_title(prefix: str, item: str) -> str:
title = f"{prefix} {clean_text(item).rstrip('.')}"
if len(title) <= 180:
return title
return f"{title[:177].rstrip()}..."
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
seen: set[tuple[str, str]] = set()
unique: list[Candidate] = []
for candidate in candidates:
key = (candidate.repo_key, normalize_title(candidate.title))
if key in seen:
continue
seen.add(key)
unique.append(candidate)
return unique
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
counts: dict[str, int] = {}
for candidate in candidates:
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
return dict(sorted(counts.items()))
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
target = infer_target(repo.root)
client = GiteaClient(target, token)
label_ids = label_ids_by_name(client, target.owner, target.repo)
if apply:
for name, color, description in GENERIC_LABELS:
if name in label_ids:
continue
created = client.request_json(
"POST",
repo_path(target.owner, target.repo, "/labels"),
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
)
label_ids[str(created["name"])] = int(created["id"])
print(f"created label {target.owner}/{target.repo}:{name}")
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
if missing and apply:
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
fingerprints: set[str] = set()
titles: set[str] = set()
for issue in issues:
body = str(issue.get("body") or "")
titles.add(normalize_title(str(issue.get("title") or "")))
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
def ensure_milestone(state: RepoState, title: str) -> int:
if title in state.milestone_ids:
return state.milestone_ids[title]
created = state.client.request_json(
"POST",
repo_path(state.target.owner, state.target.repo, "/milestones"),
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
)
state.milestone_ids[title] = int(created["id"])
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
return state.milestone_ids[title]
def normalize_title(title: str) -> str:
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
by_repo: dict[str, list[Candidate]] = {}
for candidate in candidates:
by_repo.setdefault(candidate.repo_key, []).append(candidate)
for repo_key, items in sorted(by_repo.items()):
print(f"{repo_key}:")
for item in items[:limit_per_repo]:
print(f" {item.title} [{item.priority}; {item.milestone}]")
if len(items) > limit_per_repo:
print(f" ... {len(items) - limit_per_repo} more")
if __name__ == "__main__":
raise SystemExit(main())

113
tools/gitea/gitea-install-workflow.py Normal file
View File

@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Install Gitea issue workflow files into one or more repositories."""
from __future__ import annotations
import argparse
import pathlib
import sys
from gitea_common import GiteaError, repo_root
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[2]
WORKFLOW_FILES = (
".gitea/PULL_REQUEST_TEMPLATE.md",
".gitea/ISSUE_TEMPLATE/bug_report.md",
".gitea/ISSUE_TEMPLATE/config.yaml",
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
".gitea/ISSUE_TEMPLATE/feature_request.md",
".gitea/ISSUE_TEMPLATE/task.md",
".gitea/ISSUE_TEMPLATE/tech_debt.md",
)
MODULE_LABEL_BY_REPO = {
"govoplan-core": "module/core",
"govoplan-access": "module/access",
"govoplan-admin": "module/admin",
"govoplan-tenancy": "module/tenancy",
"govoplan-policy": "module/policy",
"govoplan-audit": "module/audit",
"govoplan-mail": "module/mail",
"govoplan-files": "module/files",
"govoplan-campaign": "module/campaign",
"addideas-govoplan-website": "area/marketing",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
args = parser.parse_args()
try:
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
unique_roots = list(dict.fromkeys(target_roots))
if args.module_label and len(unique_roots) != 1:
raise GiteaError("--module-label can only be used with a single target")
rel_paths = list(WORKFLOW_FILES)
if args.include_labels_file:
rel_paths.append("docs/gitea-labels.json")
for target_root in unique_roots:
module_label = args.module_label or infer_module_label(target_root)
install_files(target_root, rel_paths, module_label, apply=args.apply)
if not args.apply:
print("Dry run only. Re-run with --apply to write files.")
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
def infer_module_label(target_root: pathlib.Path) -> str:
repo_name = target_root.name
try:
repo_name = target_root.resolve().name
except OSError:
pass
label = MODULE_LABEL_BY_REPO.get(repo_name)
if label:
return label
if repo_name.startswith("govoplan-"):
suffix = repo_name.removeprefix("govoplan-")
if suffix:
return f"module/{suffix}"
raise GiteaError(f"cannot infer repository label for {target_root}. Use --module-label module/<name> or area/<name>.")
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
print(f"Target: {target_root} ({module_label})")
for rel_path in rel_paths:
source = SOURCE_ROOT / rel_path
target = target_root / rel_path
if not source.exists():
raise GiteaError(f"missing workflow source file: {source}")
content = source.read_text(encoding="utf-8")
content = transform_content(rel_path, content, module_label)
existing = target.read_text(encoding="utf-8") if target.exists() else None
if existing == content:
print(f"unchanged {rel_path}")
continue
action = "create" if existing is None else "update"
print(f"{action if apply else 'would ' + action} {rel_path}")
if apply:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
def transform_content(rel_path: str, content: str, module_label: str) -> str:
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
return content.replace(" - module/core", f" - {module_label}")
return content
if __name__ == "__main__":
raise SystemExit(main())

250
tools/gitea/gitea-migrate-org-labels.py Normal file
View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Move issue labels from repository-local labels to organization labels."""
from __future__ import annotations
import argparse
import os
import pathlib
import re
import sys
from typing import Any
from gitea_common import (
GiteaClient,
GiteaError,
infer_target,
load_dotenv,
org_path,
repo_path,
repo_root,
require_token,
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
parser.add_argument(
"--issue-mode",
choices=("replace", "delete-add"),
default="replace",
help="replace labels in one request, or delete local ids before adding org ids",
)
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
args = parser.parse_args()
try:
root = repo_root(args.root)
load_dotenv(args.env_file or root / ".env")
target = infer_target(root, args.remote)
owner = args.org or target.owner
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
if client.token is None:
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
repo_filter = re.compile(args.repo_regex)
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
if not org_labels:
raise GiteaError(f"{owner} has no organization labels")
repos = _selected_repositories(client, owner, args.repo, repo_filter)
totals = Totals()
print(f"Organization: {owner}")
print(f"Repositories: {len(repos)}")
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
print(f"Delete repository labels: {args.delete_repo_labels}")
for index, repo in enumerate(repos, start=1):
repo_result = migrate_repository(
client,
owner=owner,
repo=repo,
org_labels=org_labels,
issue_mode=args.issue_mode,
apply=args.apply,
delete_repo_labels=args.delete_repo_labels,
)
totals.add(repo_result)
if repo_result.has_work:
print(
f"[{index}/{len(repos)}] {repo}: "
f"issue-labels={repo_result.issue_label_migrations}, "
f"delete-labels={repo_result.repo_label_deletions}, "
f"errors={len(repo_result.errors)}"
)
for error in repo_result.errors:
print(f" error: {error}")
else:
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
print("Summary:")
print(f" repositories processed: {totals.repositories}")
print(f" repositories with matching local labels: {totals.repositories_with_work}")
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
print(f" repository labels deleted: {totals.repo_label_deletions}")
print(f" errors: {totals.errors}")
if totals.errors:
return 1
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
class RepoResult:
def __init__(self) -> None:
self.issue_label_migrations = 0
self.repo_label_deletions = 0
self.errors: list[str] = []
self.matched_local_labels = 0
@property
def has_work(self) -> bool:
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
class Totals:
def __init__(self) -> None:
self.repositories = 0
self.repositories_with_work = 0
self.issue_label_migrations = 0
self.repo_label_deletions = 0
self.errors = 0
def add(self, result: RepoResult) -> None:
self.repositories += 1
if result.has_work:
self.repositories_with_work += 1
self.issue_label_migrations += result.issue_label_migrations
self.repo_label_deletions += result.repo_label_deletions
self.errors += len(result.errors)
def migrate_repository(
client: GiteaClient,
*,
owner: str,
repo: str,
org_labels: dict[str, dict[str, Any]],
issue_mode: str,
apply: bool,
delete_repo_labels: bool,
) -> RepoResult:
result = RepoResult()
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
local_labels = {
str(label.get("name") or ""): label
for label in repo_labels
if str(label.get("name") or "") in org_labels
}
result.matched_local_labels = len(local_labels)
if not local_labels:
return result
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
for issue_type in ("issues", "pulls"):
issues = client.paginate(
repo_path(owner, repo, "/issues"),
query={"state": "all", "type": issue_type},
limit=100,
)
for issue in issues:
issue_number = int(issue.get("number") or issue.get("index"))
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
final_label_ids = list(issue_label_ids)
changed = False
migrated_count = 0
local_ids_to_remove: list[int] = []
org_ids_to_add: list[int] = []
for label in issue.get("labels") or []:
local_id = _label_id(label)
if local_id is None or local_id not in local_by_id:
continue
name = str(label.get("name") or "")
org_id = org_by_name.get(name)
if org_id is None:
continue
local_ids_to_remove.append(local_id)
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
org_ids_to_add.append(org_id)
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
if org_id not in final_label_ids:
final_label_ids.append(org_id)
changed = True
migrated_count += 1
if changed:
if apply:
if issue_mode == "delete-add":
for local_id in local_ids_to_remove:
_remove_issue_label(client, owner, repo, issue_number, local_id)
if org_ids_to_add:
client.request_json(
"POST",
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
body={"labels": org_ids_to_add},
)
else:
client.request_json(
"PUT",
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
body={"labels": final_label_ids},
)
result.issue_label_migrations += migrated_count
if delete_repo_labels:
for name, label in sorted(local_labels.items()):
label_id = _label_id(label)
if label_id is None:
result.errors.append(f"{name} has no label id")
continue
if apply:
try:
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
except GiteaError as exc:
result.errors.append(f"delete repository label {name}: {exc}")
continue
result.repo_label_deletions += 1
return result
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
if explicit:
return sorted(set(explicit))
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
return [name for name in names if repo_filter.search(name)]
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
def _label_id(label: dict[str, Any]) -> int | None:
value = label.get("id") or label.get("index")
if value is None:
return None
return int(value)
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
try:
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
except GiteaError as exc:
if "HTTP 404" in str(exc):
return
raise
if __name__ == "__main__":
raise SystemExit(main())

180
tools/gitea/gitea-sync-labels.py Normal file
View File

@@ -0,0 +1,180 @@
#!/usr/bin/env python3
"""Synchronize issue labels to a Gitea repository."""
from __future__ import annotations
import argparse
import os
import pathlib
import sys
from typing import Any
from gitea_common import (
GiteaClient,
GiteaError,
infer_target,
load_dotenv,
load_json,
org_path,
repo_path,
repo_root,
require_token,
)
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[2] / "docs" / "gitea-labels.json"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument(
"--scope",
choices=("repository", "organization"),
default="repository",
help="sync repository labels or organization labels",
)
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
args = parser.parse_args()
try:
root = repo_root(args.root)
load_dotenv(args.env_file or root / ".env")
target = infer_target(root, args.remote)
labels = _load_labels(args.labels_file)
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
org_name = args.org or target.owner
if args.scope == "organization":
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
else:
print(f"Target: {target.display}")
print(f"Labels file: {args.labels_file}")
if not args.apply and not token:
print("Dry run without GITEA_TOKEN: API comparison skipped.")
for label in labels:
print(f"would ensure {label['name']} ({label['color']})")
return 0
client = GiteaClient(target, token)
if args.scope == "organization":
existing = _org_labels_by_name(client, org_name)
else:
existing = _repo_labels_by_name(client, target.owner, target.repo)
creates: list[dict[str, Any]] = []
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
for label in labels:
current = existing.get(label["name"])
if current is None:
creates.append(label)
continue
patch = _diff_label(current, label)
if patch:
updates.append((current, patch))
if not creates and not updates:
print("No label changes needed.")
return 0
for label in creates:
print(f"{'create' if args.apply else 'would create'} {label['name']}")
if args.apply:
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
for current, patch in updates:
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
if args.apply:
label_id = current.get("id") or current.get("index")
if label_id is None:
raise GiteaError(f"{current['name']} has no label id in API response")
client.request_json(
"PATCH",
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
body=patch,
)
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
payload = load_json(path)
if not isinstance(payload, list):
raise GiteaError(f"{path} must contain a JSON list")
labels: list[dict[str, Any]] = []
seen: set[str] = set()
for index, item in enumerate(payload, start=1):
if not isinstance(item, dict):
raise GiteaError(f"label #{index} must be an object")
label = {
"name": str(item.get("name", "")).strip(),
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
"description": str(item.get("description", "")).strip(),
"exclusive": bool(item.get("exclusive", False)),
}
if not label["name"]:
raise GiteaError(f"label #{index} is missing name")
if label["name"] in seen:
raise GiteaError(f"duplicate label name: {label['name']}")
if not _is_hex_color(label["color"]):
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
seen.add(label["name"])
labels.append(label)
return labels
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
labels = client.paginate(repo_path(owner, repo, "/labels"))
return {str(label.get("name")): label for label in labels}
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
labels = client.paginate(org_path(owner, "/labels"))
return {str(label.get("name")): label for label in labels}
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
if scope == "organization":
return org_path(org, "/labels")
return repo_path(owner, repo, "/labels")
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
if scope == "organization":
return org_path(org, f"/labels/{label_id}")
return repo_path(owner, repo, f"/labels/{label_id}")
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
patch: dict[str, Any] = {}
if _normalize_color(current.get("color")) != desired["color"]:
patch["color"] = desired["color"]
if str(current.get("description") or "").strip() != desired["description"]:
patch["description"] = desired["description"]
if bool(current.get("exclusive", False)) != desired["exclusive"]:
patch["exclusive"] = desired["exclusive"]
return patch
def _normalize_color(value: Any) -> str:
return str(value or "").strip().lstrip("#").lower()
def _is_hex_color(value: str) -> bool:
if len(value) != 6:
return False
return all(char in "0123456789abcdef" for char in value)
if __name__ == "__main__":
raise SystemExit(main())

637
tools/gitea/gitea-sync-wiki.py Normal file
View File

@@ -0,0 +1,637 @@
#!/usr/bin/env python3
"""Mirror project documentation files into Gitea repository wikis."""
from __future__ import annotations
import argparse
import base64
import dataclasses
import hashlib
import os
import pathlib
import re
import shutil
import subprocess
import sys
import urllib.parse
from typing import Any
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
MANAGED_MARKER = "<!-- codex-wiki-sync:"
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
EXCLUDED_PARTS = {
".git",
".gitea",
".venv",
"node_modules",
"__pycache__",
"dist",
"build",
".cache",
".module-test-build",
}
EXCLUDED_NAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"uv.lock",
}
@dataclasses.dataclass(frozen=True)
class RepoInfo:
root: pathlib.Path
owner: str
repo: str
@property
def key(self) -> str:
return self.root.name
@dataclasses.dataclass(frozen=True)
class WikiSource:
repo_key: str
path: pathlib.Path
origin: str
page_title: str
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
parser.add_argument("--apply", action="store_true")
args = parser.parse_args()
try:
load_dotenv(args.env_file)
repos = discover_hosted_repos(args.git_root)
if args.repo:
requested = set(args.repo)
repos = {key: repo for key, repo in repos.items() if key in requested}
missing = requested - set(repos)
if missing:
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
sources = discover_sources(repos, args.products_root)
if args.page:
requested_pages = set(args.page)
sources = [source for source in sources if source.page_title in requested_pages]
missing_pages = requested_pages - {source.page_title for source in sources}
if missing_pages:
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
print(f"Hosted repositories: {len(repos)}")
print(f"Wiki source files: {len(sources)}")
for repo_key, count in grouped_counts(sources).items():
print(f" {repo_key}: {count}")
if not args.apply:
preview_sources(sources)
print("Dry run only. Re-run with --apply to write wiki pages.")
return 0
token = require_token() if args.transport == "api" else ""
failures = 0
for repo_key, repo_sources in group_sources(sources).items():
repo = repos[repo_key]
try:
if args.transport == "git":
sync_repo_wiki_git(
repo,
repo_sources,
cache_dir=args.wiki_cache_dir,
overwrite_unmanaged=args.overwrite_unmanaged,
commit_message=args.commit_message,
write_index=not bool(args.page),
prune_managed=args.prune_managed and not bool(args.page),
)
else:
sync_repo_wiki(
repo,
repo_sources,
token,
overwrite_unmanaged=args.overwrite_unmanaged,
write_index=not bool(args.page),
prune_managed=args.prune_managed and not bool(args.page),
)
except GiteaError as exc:
failures += 1
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
return 1 if failures else 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
repos: dict[str, RepoInfo] = {}
for gitdir in sorted(git_root.glob("*/.git")):
root = gitdir.parent
result = subprocess.run(
["git", "-C", str(root), "remote", "get-url", "origin"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
if "git.add-ideas.de" not in result.stdout:
continue
target = infer_target(root)
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
return repos
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
sources: list[WikiSource] = []
for repo in repos.values():
for path in repo.root.rglob("*"):
if is_repo_doc(repo.root, path):
rel = path.relative_to(repo.root)
sources.append(
WikiSource(
repo_key=repo.key,
path=path,
origin="repository",
page_title=title_for_path("Repo", rel),
)
)
product_map = {
"co2api": "emission-api-lib",
"govoplan": "govoplan-core",
"meubility": "meubility-workbench",
"mousehold": "mousehold",
"prvnncr": "prvnncr-server",
}
if products_root.exists():
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
repo_key = product_map.get(product_dir.name)
if not repo_key or repo_key not in repos:
continue
for path in product_dir.rglob("*"):
if is_product_doc(path):
rel = path.relative_to(product_dir)
sources.append(
WikiSource(
repo_key=repo_key,
path=path,
origin=f"product:{product_dir.name}",
page_title=title_for_path(f"Product {product_dir.name}", rel),
)
)
unique: dict[tuple[str, str], WikiSource] = {}
for source in sources:
unique[(source.repo_key, source.page_title)] = source
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
if not path.is_file() or not is_text_file(path):
return False
parts = set(path.relative_to(repo_root_path).parts)
if parts & EXCLUDED_PARTS:
return False
if path.name in EXCLUDED_NAMES:
return False
if SENSITIVE_NAME_RE.search(path.name):
return False
rel = path.relative_to(repo_root_path)
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
return True
if "generated" in rel.parts:
return False
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
return True
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
return True
return False
def is_product_doc(path: pathlib.Path) -> bool:
if not path.is_file() or not is_text_file(path):
return False
if path.name in EXCLUDED_NAMES:
return False
if SENSITIVE_NAME_RE.search(path.name):
return False
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
return False
if re.search(
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
path.name,
re.IGNORECASE,
) and path.suffix.lower() in {".md", ".txt"}:
return True
return False
def is_text_file(path: pathlib.Path) -> bool:
if path.suffix.lower() not in TEXT_SUFFIXES:
return False
try:
chunk = path.read_bytes()[:4096]
except OSError:
return False
return b"\x00" not in chunk
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
stem = rel.with_suffix("").as_posix()
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
return f"{prefix}-{stem}"[:180]
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
grouped: dict[str, list[WikiSource]] = {}
for source in sources:
grouped.setdefault(source.repo_key, []).append(source)
return dict(sorted(grouped.items()))
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
def preview_sources(sources: list[WikiSource]) -> None:
for repo_key, repo_sources in group_sources(sources).items():
print(f"{repo_key}:")
for source in repo_sources[:80]:
print(f" {source.page_title} <- {source.path}")
if len(repo_sources) > 80:
print(f" ... {len(repo_sources) - 80} more")
def sync_repo_wiki(
repo: RepoInfo,
sources: list[WikiSource],
token: str,
*,
overwrite_unmanaged: bool,
write_index: bool = True,
prune_managed: bool = False,
) -> None:
target = infer_target(repo.root)
client = GiteaClient(target, token)
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
existing_page_names = {
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
for page in existing_pages
if page.get("title")
}
index_lines = [
f"# {target.repo} Project Wiki Index",
"",
f"{MANAGED_MARKER}index -->",
"",
"This page is generated from repository and product-directory project files.",
"",
]
for source in sources:
content = render_wiki_content(source)
changed = put_wiki_page(
client,
target.owner,
target.repo,
source.page_title,
content,
existing_page_names,
overwrite_unmanaged=overwrite_unmanaged,
)
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
if write_index:
put_wiki_page(
client,
target.owner,
target.repo,
"Codex-Project-Index",
"\n".join(index_lines) + "\n",
existing_page_names,
overwrite_unmanaged=True,
)
else:
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
if prune_managed and write_index:
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
def sync_repo_wiki_git(
repo: RepoInfo,
sources: list[WikiSource],
*,
cache_dir: pathlib.Path,
overwrite_unmanaged: bool,
commit_message: str,
write_index: bool = True,
prune_managed: bool = False,
) -> None:
target = infer_target(repo.root)
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
index_lines = [
f"# {target.repo} Project Wiki Index",
"",
f"{MANAGED_MARKER}index -->",
"",
"This page is generated from repository and product-directory project files.",
"",
]
for source in sources:
content = render_wiki_content(source)
status = write_wiki_page_file(
wiki_root,
source.page_title,
content,
overwrite_unmanaged=overwrite_unmanaged,
)
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
if write_index:
write_wiki_page_file(
wiki_root,
"Codex-Project-Index",
"\n".join(index_lines) + "\n",
overwrite_unmanaged=True,
)
else:
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
if prune_managed and write_index:
prune_managed_wiki_pages_git(wiki_root, sources)
if not git_has_changes(wiki_root):
print(f"unchanged {target.owner}/{target.repo} wiki repository")
return
run_git(wiki_root, "add", "-A")
if not git_has_staged_changes(wiki_root):
print(f"unchanged {target.owner}/{target.repo} wiki repository")
return
ensure_git_identity(wiki_root)
run_git(wiki_root, "commit", "-m", commit_message)
run_git(wiki_root, "push")
print(f"pushed {target.owner}/{target.repo} wiki repository")
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
remote = wiki_remote_for_repo(repo.root)
cache_dir.mkdir(parents=True, exist_ok=True)
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
if (checkout / ".git").exists():
run_git(checkout, "remote", "set-url", "origin", remote)
run_git(checkout, "fetch", "origin")
branch = current_branch(checkout)
if branch:
run_git(checkout, "reset", "--hard", f"origin/{branch}")
else:
run_git(checkout, "reset", "--hard")
run_git(checkout, "clean", "-fd")
return checkout
if checkout.exists():
shutil.rmtree(checkout)
result = subprocess.run(
["git", "clone", remote, str(checkout)],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode == 0:
return checkout
checkout.mkdir(parents=True, exist_ok=True)
run_git(checkout, "init")
run_git(checkout, "remote", "add", "origin", remote)
run_git(checkout, "checkout", "-b", "master")
return checkout
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
result = subprocess.run(
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0 or not result.stdout.strip():
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
remote = result.stdout.strip()
if remote.endswith(".git"):
return f"{remote[:-4]}.wiki.git"
return f"{remote}.wiki.git"
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
path = wiki_root / f"{wiki_file_stem(title)}.md"
if path.exists():
current = read_text(path)
if current == content:
return "unchanged"
if MANAGED_MARKER not in current and not overwrite_unmanaged:
return "skipped unmanaged"
path.write_text(content, encoding="utf-8")
return "updated"
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
for path in sorted(wiki_root.glob("*.md")):
if path.name in desired:
continue
current = read_text(path)
if MANAGED_MARKER not in current:
continue
path.unlink()
print(f"deleted stale managed wiki page {path.name}")
def prune_managed_wiki_pages_api(
client: GiteaClient,
owner: str,
repo: str,
existing_page_names: dict[str, str],
sources: list[WikiSource],
) -> None:
desired = {source.page_title for source in sources}
desired.add("Codex-Project-Index")
for title, page_name in sorted(existing_page_names.items()):
if title in desired:
continue
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
if MANAGED_MARKER not in decode_wiki_content(current):
continue
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
def wiki_file_stem(title: str) -> str:
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
def git_has_changes(root: pathlib.Path) -> bool:
result = subprocess.run(
["git", "-C", str(root), "status", "--porcelain"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
return bool(result.stdout.strip())
def git_has_staged_changes(root: pathlib.Path) -> bool:
result = subprocess.run(
["git", "-C", str(root), "diff", "--cached", "--quiet"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode == 0:
return False
if result.returncode == 1:
return True
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
def current_branch(root: pathlib.Path) -> str:
result = subprocess.run(
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
branch = result.stdout.strip()
if result.returncode != 0 or branch == "HEAD":
return ""
return branch
def ensure_git_identity(root: pathlib.Path) -> None:
if not git_config_value(root, "user.email"):
run_git(root, "config", "user.email", "codex@govoplan.local")
if not git_config_value(root, "user.name"):
run_git(root, "config", "user.name", "Codex")
def git_config_value(root: pathlib.Path, key: str) -> str:
result = subprocess.run(
["git", "-C", str(root), "config", "--get", key],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
return result.stdout.strip() if result.returncode == 0 else ""
def run_git(root: pathlib.Path, *args: str) -> None:
result = subprocess.run(
["git", "-C", str(root), *args],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
command = "git -C " + str(root) + " " + " ".join(args)
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
try:
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
except GiteaError as exc:
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
return []
raise
def render_wiki_content(source: WikiSource) -> str:
raw = read_text(source.path)
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
header = [
f"{MANAGED_MARKER}{fingerprint} -->",
"",
f"> Mirrored from `{source.path}`.",
f"> Origin: `{source.origin}`.",
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
"",
"---",
"",
]
if source.path.suffix.lower() == ".csv":
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
return "\n".join(header) + raw.rstrip() + "\n"
def put_wiki_page(
client: GiteaClient,
owner: str,
repo: str,
title: str,
content: str,
existing_page_names: dict[str, str],
*,
overwrite_unmanaged: bool,
) -> bool:
body = {
"title": title,
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
"message": f"Sync {title} from project files",
}
if title in existing_page_names:
page_name = existing_page_names[title]
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
current_content = decode_wiki_content(current)
if current_content == content:
return False
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
return False
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
return True
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
existing_page_names[title] = title
return True
def quote_wiki_page_name(value: str) -> str:
return urllib.parse.quote(value, safe="+")
def decode_wiki_content(page: dict[str, Any]) -> str:
encoded = str(page.get("content_base64") or "")
if not encoded:
return ""
return base64.b64decode(encoded).decode("utf-8")
def read_text(path: pathlib.Path) -> str:
try:
return path.read_text(encoding="utf-8")
except UnicodeDecodeError:
return path.read_text(encoding="latin-1")
if __name__ == "__main__":
raise SystemExit(main())

377
tools/gitea/gitea-todo-import.py Normal file
View File

@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""Preview or import inline TODO-style markers as Gitea issues."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import pathlib
import re
import subprocess
import sys
from typing import Any
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
MARKER_RE = re.compile(
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
re.IGNORECASE,
)
DEFAULT_EXCLUDES = (
"!**/.git/**",
"!**/.venv/**",
"!**/node_modules/**",
"!**/__pycache__/**",
"!**/.module-test-build/**",
"!**/dist/**",
"!**/build/**",
"!**/.cache/**",
"!.gitea/**",
"!docs/GITEA_ISSUES.md",
"!tools/gitea/gitea-todo-import.py",
"!tools/gitea/gitea-sync-labels.py",
"!tools/gitea/gitea-codex-note.py",
"!tools/gitea/gitea_common.py",
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
parser.add_argument(
"--module-label",
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
)
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
args = parser.parse_args()
try:
root = repo_root(args.root)
load_dotenv(args.env_file or root / ".env")
target = infer_target(root, args.remote)
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
markers = scan_markers(root, include_linked=args.include_linked)
if args.limit > 0:
markers = markers[: args.limit]
module_label = args.module_label or infer_module_label(root)
previews = [
build_issue_preview(
target.owner,
target.repo,
marker,
module_label=module_label,
infer_areas=not args.no_area_labels,
extra_labels=args.extra_label,
)
for marker in markers
]
if args.json:
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
else:
print(f"Target: {target.display}")
print(f"Scanned root: {root}")
print(f"Found {len(previews)} importable marker(s).")
for preview in previews:
print(f"- {preview.title}")
print(f" {preview.location}")
print(f" labels: {', '.join(preview.labels)}")
if not args.apply:
if not previews:
return 0
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
return 0
if not previews:
print("No issues to create.")
return 0
client = GiteaClient(target, token)
label_ids = load_label_ids(client, target.owner, target.repo)
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
if missing_labels:
joined = ", ".join(missing_labels)
raise GiteaError(f"missing labels: {joined}. Run tools/gitea/gitea-sync-labels.py --apply first.")
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
created = 0
skipped = 0
for preview in previews:
if preview.fingerprint in existing_fingerprints:
print(f"skip existing {preview.location}")
skipped += 1
continue
issue = client.request_json(
"POST",
repo_path(target.owner, target.repo, "/issues"),
body={
"title": preview.title,
"body": preview.body,
"labels": [label_ids[label] for label in preview.labels],
},
)
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
created += 1
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
return 0
except GiteaError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
class Marker:
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
self.path = path
self.line = line
self.column = column
self.marker = marker.upper()
self.context = context
self.text = text
self.raw = raw.rstrip("\n")
@property
def location(self) -> str:
return f"{self.path}:{self.line}"
class IssuePreview:
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
self.title = title
self.body = body
self.labels = labels
self.fingerprint = fingerprint
self.location = location
def as_dict(self) -> dict[str, Any]:
return {
"title": self.title,
"body": self.body,
"labels": self.labels,
"fingerprint": self.fingerprint,
"location": self.location,
}
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
command = [
"rg",
"--json",
"--hidden",
"--line-number",
"--column",
"-e",
r"\b(TODO|FIXME|XXX|HACK)\b",
]
for pattern in DEFAULT_EXCLUDES:
command.extend(["--glob", pattern])
command.append(str(root))
result = subprocess.run(
command,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode not in {0, 1}:
raise GiteaError(f"rg failed: {result.stderr.strip()}")
markers: list[Marker] = []
for line in result.stdout.splitlines():
event = json.loads(line)
if event.get("type") != "match":
continue
data = event["data"]
raw_line = data["lines"]["text"]
match = MARKER_RE.search(raw_line)
if not match:
continue
if not _looks_like_marker(match):
continue
context = (match.group("context") or "").strip()
if not include_linked and _already_linked(context, raw_line):
continue
markers.append(
Marker(
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
line=int(data["line_number"]),
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
marker=match.group("marker"),
context=context,
text=clean_marker_text(match.group("text") or raw_line),
raw=raw_line,
)
)
return markers
def build_issue_preview(
owner: str,
repo: str,
marker: Marker,
*,
module_label: str,
infer_areas: bool,
extra_labels: list[str],
) -> IssuePreview:
fingerprint = marker_fingerprint(owner, repo, marker)
summary = marker.text or f"review {marker.location}"
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
body = "\n".join(
[
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
"",
"Imported from an inline marker.",
"",
f"- Repository: `{owner}/{repo}`",
f"- Location: `{marker.location}`",
f"- Marker: `{marker.marker}`",
f"- Context: `{marker.context or 'none'}`",
"",
"Current line:",
"",
"```text",
marker.raw,
"```",
"",
"Migration rule:",
"",
"- Keep this Gitea issue as the canonical backlog item.",
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
]
)
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
def clean_marker_text(text: str) -> str:
cleaned = text.strip()
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
return cleaned
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
area = infer_area_label(marker.path) if infer_areas else ""
if area:
labels.append(area)
return labels
def marker_type_label(marker: str) -> str:
if marker == "FIXME":
return "type/bug"
if marker in {"HACK", "XXX"}:
return "type/debt"
return "type/task"
def infer_area_label(path: pathlib.Path) -> str:
text = path.as_posix()
if text.startswith("webui/"):
return "area/webui"
if text.startswith("alembic/"):
return "area/migrations"
if text.startswith("docs/"):
return "area/docs"
if text.startswith("scripts/") or text.startswith("tools/"):
return "area/devex"
if "/rbac" in text or "permission" in text:
return "area/rbac"
if "/access" in text or "auth" in text or "session" in text:
return "area/auth"
if "tenant" in text:
return "area/tenancy"
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
return "area/governance"
if "module" in text:
return "area/module-system"
if "migration" in text:
return "area/migrations"
if "router" in text or "api" in text:
return "area/api"
if "db" in text or "model" in text or "persistence" in text:
return "area/db"
return ""
def infer_module_label(root: pathlib.Path) -> str:
known = {
"govoplan-core": "module/core",
"govoplan-access": "module/access",
"govoplan-admin": "module/admin",
"govoplan-tenancy": "module/tenancy",
"govoplan-policy": "module/policy",
"govoplan-audit": "module/audit",
"govoplan-mail": "module/mail",
"govoplan-files": "module/files",
"govoplan-campaign": "module/campaign",
"addideas-govoplan-website": "area/marketing",
}
if root.name in known:
return known[root.name]
if root.name.startswith("govoplan-"):
suffix = root.name.removeprefix("govoplan-")
if suffix:
return f"module/{suffix}"
return "module/core"
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
def truncate_title(title: str, limit: int = 180) -> str:
if len(title) <= limit:
return title
return f"{title[: limit - 1].rstrip()}..."
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
return label_ids_by_name(client, owner, repo)
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
issues = client.paginate(
repo_path(owner, repo, "/issues"),
query={"state": "all", "labels": "source/todo-scan"},
)
fingerprints: set[str] = set()
for issue in issues:
body = str(issue.get("body") or "")
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
fingerprints.add(match.group(1))
return fingerprints
def _already_linked(context: str, raw_line: str) -> bool:
linked_text = f"{context} {raw_line}".lower()
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
def _looks_like_marker(match: re.Match[str]) -> bool:
text = (match.group("text") or "").strip()
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
if __name__ == "__main__":
raise SystemExit(main())

290
tools/gitea/gitea_common.py Normal file
View File

@@ -0,0 +1,290 @@
#!/usr/bin/env python3
"""Shared helpers for Gitea maintenance scripts."""
from __future__ import annotations
import dataclasses
import json
import os
import pathlib
import re
import subprocess
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
class GiteaError(RuntimeError):
"""Raised when a Gitea API request fails."""
@dataclasses.dataclass(frozen=True)
class RepoTarget:
base_url: str
owner: str
repo: str
@property
def display(self) -> str:
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
@property
def api_base(self) -> str:
return f"{self.base_url.rstrip('/')}/api/v1"
def repo_root(start: pathlib.Path) -> pathlib.Path:
result = subprocess.run(
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
return start.resolve()
return pathlib.Path(result.stdout.strip()).resolve()
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
env_url = os.environ.get("GITEA_URL")
env_owner = os.environ.get("GITEA_OWNER")
env_repo = os.environ.get("GITEA_REPO")
remote_url = _git_remote(root, remote_name)
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
base_url = (env_url or inferred_url).rstrip("/")
owner = env_owner or inferred_owner
repo = env_repo or inferred_repo
missing = [
name
for name, value in (
("GITEA_URL", base_url),
("GITEA_OWNER", owner),
("GITEA_REPO", repo),
)
if not value
]
if missing:
joined = ", ".join(missing)
raise GiteaError(
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
)
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
def quote_path(value: str) -> str:
return urllib.parse.quote(value, safe="")
class GiteaClient:
def __init__(self, target: RepoTarget, token: str | None) -> None:
self.target = target
self.token = token
def request_json(
self,
method: str,
path: str,
*,
body: dict[str, Any] | None = None,
query: dict[str, Any] | None = None,
) -> Any:
url = f"{self.target.api_base}{path}"
if query:
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
data = None
headers = {
"Accept": "application/json",
"User-Agent": "codex-gitea-maintenance",
}
if self.token:
headers["Authorization"] = f"token {self.token}"
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(request, timeout=30) as response:
payload = response.read().decode("utf-8")
if not payload:
return None
return json.loads(payload)
except urllib.error.HTTPError as exc:
message = exc.read().decode("utf-8", errors="replace")
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
except urllib.error.URLError as exc:
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
def paginate(
self,
path: str,
*,
query: dict[str, Any] | None = None,
limit: int = 100,
) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
page = 1
effective_limit = min(limit, 50)
seen_pages: set[tuple[Any, ...]] = set()
while True:
page_query = dict(query or {})
page_query.update({"page": page, "limit": effective_limit})
payload = self.request_json("GET", path, query=page_query)
if not isinstance(payload, list):
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
if not payload:
return items
signature = tuple(
item.get("id") or item.get("number") or item.get("index") or item.get("name")
for item in payload
if isinstance(item, dict)
)
if signature in seen_pages:
return items
seen_pages.add(signature)
items.extend(payload)
if len(payload) < effective_limit:
return items
page += 1
def load_json(path: pathlib.Path) -> Any:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
def load_dotenv(path: pathlib.Path | None) -> list[str]:
if path is None or not path.exists():
return []
loaded: list[str] = []
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
if stripped.startswith("export "):
stripped = stripped[len("export ") :].strip()
if "=" not in stripped:
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
key, value = stripped.split("=", 1)
key = key.strip()
value = _strip_env_value(value.strip())
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
if key not in os.environ:
os.environ[key] = value
loaded.append(key)
return loaded
def require_token() -> str:
token = os.environ.get("GITEA_TOKEN")
if not token:
raise GiteaError(
"GITEA_TOKEN is required when using --apply. "
"Set it in the environment, the target .env, or --env-file."
)
return token
def repo_path(owner: str, repo: str, suffix: str) -> str:
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
def org_path(owner: str, suffix: str) -> str:
return f"/orgs/{quote_path(owner)}{suffix}"
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
"""Return org labels plus optional repository labels, keyed by label name.
Gitea stores organization labels separately from repository labels. Issue
APIs can use label ids from both scopes on supported instances, so issue
creation helpers should resolve both. Repository labels intentionally win
when a repo carries a local label with the same name.
"""
labels: dict[str, dict[str, Any]] = {}
try:
for label in client.paginate(org_path(owner, "/labels")):
name = str(label.get("name") or "")
if name:
labels[name] = label
except GiteaError:
# User-owned repositories or older instances may not expose org labels.
pass
if repo:
for label in client.paginate(repo_path(owner, repo, "/labels")):
name = str(label.get("name") or "")
if name:
labels[name] = label
return labels
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
ids: dict[str, int] = {}
for name, label in labels_by_name(client, owner, repo).items():
label_id = label.get("id") or label.get("index")
if label_id is None:
continue
ids[name] = int(label_id)
return ids
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
result = subprocess.run(
["git", "-C", str(root), "remote", "get-url", remote_name],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
return ""
return result.stdout.strip()
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
if not remote_url:
return "", "", ""
parsed = urllib.parse.urlparse(remote_url)
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
owner, repo = _owner_repo_from_parts(path_parts)
if parsed.scheme in {"http", "https"}:
prefix = "/".join(path_parts[:-2])
base_path = f"/{prefix}" if prefix else ""
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
if scp_like:
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
owner, repo = _owner_repo_from_parts(path_parts)
return f"https://{scp_like.group('host')}", owner, repo
return "", "", ""
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
if len(path_parts) < 2:
return "", ""
return path_parts[-2], _strip_git_suffix(path_parts[-1])
def _strip_git_suffix(value: str) -> str:
return value[:-4] if value.endswith(".git") else value
def _strip_env_value(value: str) -> str:
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
return value[1:-1]
return value

217
tools/launch/launch-dev.sh Normal file
View File

@@ -0,0 +1,217 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
WEBUI_ROOT="${GOVOPLAN_WEBUI_ROOT:-$ROOT/webui}"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
NPM="${NPM:-$NODE_BIN/npm}"
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
OPEN_BROWSER="${OPEN_BROWSER:-1}"
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
LOG_DIR="${GOVOPLAN_DEV_LOG_DIR:-$META_ROOT/runtime/dev-launcher}"
BACKEND_LOG="$LOG_DIR/backend.log"
FRONTEND_LOG="$LOG_DIR/frontend.log"
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
backend_pid=""
frontend_pid=""
run_grouped_pid=""
fail() {
printf 'launch-dev: %s\n' "$*" >&2
exit 1
}
run_grouped() {
local cwd="$1"
local log_file="$2"
shift 2
if command -v setsid >/dev/null 2>&1; then
setsid sh -c 'cd "$1" || exit 127; shift; exec "$@"' sh "$cwd" "$@" >"$log_file" 2>&1 &
else
(
cd "$cwd"
exec "$@"
) >"$log_file" 2>&1 &
fi
run_grouped_pid="$!"
}
terminate_process() {
local pid="${1:-}"
local pgid=""
[ -n "$pid" ] || return 0
kill -0 "$pid" 2>/dev/null || return 0
pgid="$(ps -o pgid= -p "$pid" 2>/dev/null | tr -d ' ' || true)"
if [ "$pgid" = "$pid" ]; then
kill -TERM "-$pid" 2>/dev/null || true
else
kill "$pid" 2>/dev/null || true
fi
sleep 1
kill -0 "$pid" 2>/dev/null || return 0
if [ "$pgid" = "$pid" ]; then
kill -KILL "-$pid" 2>/dev/null || true
else
kill -KILL "$pid" 2>/dev/null || true
fi
}
case "$DEV_DATABASE_BACKEND" in
postgres)
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
;;
sqlite)
mkdir -p "$META_ROOT/runtime"
export DATABASE_URL="${DATABASE_URL:-sqlite:///$META_ROOT/runtime/govoplan-dev.db}"
;;
*)
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
;;
esac
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
port_is_free() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
host = sys.argv[1]
port = int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((host, port))
except OSError:
raise SystemExit(1)
PY
}
wait_for_url() {
"$PYTHON" - "$1" <<'PY'
import sys
import time
import urllib.request
url = sys.argv[1]
deadline = time.monotonic() + 60
last_error = None
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if 200 <= response.status < 500:
raise SystemExit(0)
except Exception as exc: # noqa: BLE001 - printed only on timeout.
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
cleanup() {
terminate_process "$frontend_pid"
terminate_process "$backend_pid"
rm -f "$NPM_USERCONFIG"
}
trap cleanup EXIT INT TERM
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $META_ROOT && python3 -m venv .venv && ./.venv/bin/python -m pip install --upgrade pip && ./.venv/bin/python -m pip install -r requirements-dev.txt"
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
(
cd "$META_ROOT"
"$PYTHON" - <<'PY'
import importlib
importlib.import_module("govoplan_core.devserver")
PY
) || fail "Python environment at $PYTHON cannot import govoplan_core.devserver. Run: cd $META_ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
mkdir -p "$LOG_DIR"
: > "$BACKEND_LOG"
: > "$FRONTEND_LOG"
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
fi
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
run_grouped "$ROOT" "$BACKEND_LOG" "$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
backend_pid="$run_grouped_pid"
printf 'Waiting for %s/health\n' "$BACKEND_URL"
wait_for_url "$BACKEND_URL/health" || {
tail -n 80 "$BACKEND_LOG" >&2 || true
fail "backend did not become healthy"
}
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
frontend_args=(run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT" --strictPort)
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
frontend_args+=(--force)
fi
run_grouped "$WEBUI_ROOT" "$FRONTEND_LOG" \
env \
-u npm_config_tmp \
-u NPM_CONFIG_TMP \
"PATH=$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
"NPM_CONFIG_USERCONFIG=$NPM_USERCONFIG" \
"CHOKIDAR_USEPOLLING=$FRONTEND_USE_POLLING" \
"CHOKIDAR_INTERVAL=$FRONTEND_POLLING_INTERVAL" \
"VITE_API_PROXY_TARGET=$BACKEND_URL" \
"$NPM" "${frontend_args[@]}"
frontend_pid="$run_grouped_pid"
printf 'Waiting for %s\n' "$FRONTEND_URL"
wait_for_url "$FRONTEND_URL" || {
tail -n 80 "$FRONTEND_LOG" >&2 || true
fail "frontend did not become reachable"
}
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
fi
cat <<EOF
GovOPlaN is running.
Web UI: $FRONTEND_URL
API: $BACKEND_URL/api/v1
Health: $BACKEND_URL/health
DB: $DATABASE_URL
Logs:
Backend: $BACKEND_LOG
WebUI: $FRONTEND_LOG
Frontend reload:
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
Vite --force: $FRONTEND_FORCE_RELOAD
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
Press Ctrl+C to stop both processes.
EOF
wait

244
tools/launch/launch-production-like-dev.sh Normal file
View File

@@ -0,0 +1,244 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
PROFILE_ROOT="$META_ROOT/dev/production-like"
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
if [ ! -f "$PROFILE_ENV" ]; then
PROFILE_ENV="$PROFILE_ROOT/.env.example"
fi
WEBUI_ROOT="$ROOT/webui"
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
NPM="${NPM:-$NODE_BIN/npm}"
DOCKER="${DOCKER:-docker}"
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
OPEN_BROWSER="${OPEN_BROWSER:-1}"
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
LOG_DIR="$META_ROOT/runtime/production-like/logs"
BACKEND_LOG="$LOG_DIR/backend.log"
WORKER_LOG="$LOG_DIR/worker.log"
FRONTEND_LOG="$LOG_DIR/frontend.log"
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
backend_pid=""
worker_pid=""
frontend_pid=""
fail() {
printf 'launch-production-like-dev: %s\n' "$*" >&2
exit 1
}
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
set -a
# shellcheck disable=SC1090
. "$PROFILE_ENV"
set +a
export APP_ENV="${APP_ENV:-staging}"
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$META_ROOT/runtime/production-like/files}"
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
if [ -z "${MASTER_KEY_B64:-}" ]; then
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
from cryptography.fernet import Fernet
print(Fernet.generate_key().decode())
PY
)"
export MASTER_KEY_B64
fi
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
compose() {
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
}
port_is_free() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
host = sys.argv[1]
port = int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((host, port))
except OSError:
raise SystemExit(1)
PY
}
wait_for_tcp() {
"$PYTHON" - "$1" "$2" <<'PY'
import socket
import sys
import time
host = sys.argv[1]
port = int(sys.argv[2])
deadline = time.monotonic() + 90
last_error = None
while time.monotonic() < deadline:
try:
with socket.create_connection((host, port), timeout=2):
raise SystemExit(0)
except OSError as exc:
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
wait_for_url() {
"$PYTHON" - "$1" <<'PY'
import sys
import time
import urllib.request
url = sys.argv[1]
deadline = time.monotonic() + 90
last_error = None
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if 200 <= response.status < 500:
raise SystemExit(0)
except Exception as exc: # noqa: BLE001 - printed only on timeout.
last_error = exc
time.sleep(1)
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
raise SystemExit(1)
PY
}
cleanup() {
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
kill "$frontend_pid" 2>/dev/null || true
fi
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
kill "$backend_pid" 2>/dev/null || true
fi
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
kill "$worker_pid" 2>/dev/null || true
fi
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
compose down >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT INT TERM
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $META_ROOT && python3 -m venv .venv && ./.venv/bin/python -m pip install --upgrade pip && ./.venv/bin/python -m pip install -r requirements-dev.txt"
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
: > "$BACKEND_LOG"
: > "$WORKER_LOG"
: > "$FRONTEND_LOG"
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
printf 'Starting production-like dependencies through Docker Compose\n'
compose up -d
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
(
cd "$ROOT"
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
)
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
(
cd "$ROOT"
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
) >"$BACKEND_LOG" 2>&1 &
backend_pid="$!"
printf 'Waiting for %s/health\n' "$BACKEND_URL"
wait_for_url "$BACKEND_URL/health" || {
tail -n 80 "$BACKEND_LOG" >&2 || true
fail "backend did not become healthy"
}
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
(
cd "$ROOT"
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
--queues "$CELERY_QUEUES" \
--hostname "govoplan-production-like@%h" \
--loglevel INFO
) >"$WORKER_LOG" 2>&1 &
worker_pid="$!"
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
(
cd "$WEBUI_ROOT"
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
VITE_API_PROXY_TARGET="$BACKEND_URL" \
"$NPM" run dev -- --force
) >"$FRONTEND_LOG" 2>&1 &
frontend_pid="$!"
printf 'Waiting for %s\n' "$FRONTEND_URL"
wait_for_url "$FRONTEND_URL" || {
tail -n 80 "$FRONTEND_LOG" >&2 || true
fail "frontend did not become reachable"
}
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
fi
cat <<EOF
GovOPlaN production-like development profile is running.
Web UI: $FRONTEND_URL
API: $BACKEND_URL/api/v1
Health: $BACKEND_URL/health
Ops: $BACKEND_URL/api/v1/ops/status
DB: $DATABASE_URL
Redis: $REDIS_URL
Files: $FILE_STORAGE_LOCAL_ROOT
Logs:
Backend: $BACKEND_LOG
Worker: $WORKER_LOG
WebUI: $FRONTEND_LOG
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
Press Ctrl+C to stop API, worker, and WebUI.
EOF
wait

116
tools/launch/production-like-dev.sh Normal file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
PROFILE_ROOT="$META_ROOT/dev/production-like"
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
if [ ! -f "$PROFILE_ENV" ]; then
PROFILE_ENV="$PROFILE_ROOT/.env.example"
fi
VENV_ROOT="${GOVOPLAN_VENV_ROOT:-$META_ROOT/.venv}"
PYTHON="${PYTHON:-$VENV_ROOT/bin/python}"
DOCKER="${DOCKER:-docker}"
usage() {
cat <<'EOF'
Usage: tools/launch/production-like-dev.sh <command> [--yes]
Commands:
start Start the production-like API, worker, WebUI, PostgreSQL, and Redis profile.
stop Stop PostgreSQL and Redis profile containers.
status Show profile container status and validate the effective environment.
seed Run migrations and seed development data against the profile database.
reset --yes Stop containers, remove profile volumes, and remove runtime/production-like data.
validate-config Validate the profile environment only.
The start command delegates to tools/launch/launch-production-like-dev.sh. Stop/reset
operate on Docker dependencies; API/WebUI/worker processes started by the
launcher are stopped with Ctrl+C in that launcher terminal.
EOF
}
fail() {
printf 'production-like-dev: %s\n' "$*" >&2
exit 1
}
command="${1:-}"
if [ -z "$command" ] || [ "$command" = "-h" ] || [ "$command" = "--help" ]; then
usage
exit 0
fi
shift || true
yes=0
for arg in "$@"; do
case "$arg" in
--yes) yes=1 ;;
*) fail "unknown argument: $arg" ;;
esac
done
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $META_ROOT && python3 -m venv .venv && ./.venv/bin/python -m pip install --upgrade pip && ./.venv/bin/python -m pip install -r requirements-dev.txt"
set -a
# shellcheck disable=SC1090
. "$PROFILE_ENV"
set +a
export APP_ENV="${APP_ENV:-staging}"
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$META_ROOT/runtime/production-like/files}"
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}"
if [ -z "${MASTER_KEY_B64:-}" ]; then
MASTER_KEY_B64="$("$PYTHON" -m govoplan_core.commands.config env-template --profile production-like --generate-secrets | sed -n 's/^MASTER_KEY_B64=//p' | head -n 1)"
export MASTER_KEY_B64
fi
compose() {
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
}
validate_config() {
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
}
case "$command" in
start)
exec "$META_ROOT/tools/launch/launch-production-like-dev.sh"
;;
stop)
compose down
;;
status)
compose ps
validate_config
;;
seed)
validate_config
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
;;
reset)
[ "$yes" = "1" ] || fail "reset is destructive; rerun with --yes"
compose down -v
rm -rf "$META_ROOT/runtime/production-like"
;;
validate-config)
validate_config
;;
*)
usage >&2
fail "unknown command: $command"
;;
esac

9
tools/launch/start-dev-postgres.sh Normal file
View File

@@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
GOVOPLAN_CORE_ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
GOVOPLAN_CORE_ROOT="$(cd "$GOVOPLAN_CORE_ROOT" && pwd)"
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
exec $COMPOSE_BIN -f "$META_ROOT/dev/postgres/docker-compose.yml" up -d "$@"

83
tools/release/generate-catalog-keypair.py Normal file
View File

@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
from __future__ import annotations
import argparse
import base64
from datetime import UTC, datetime
import json
from pathlib import Path
import stat
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
args = parser.parse_args()
private_path = args.private_key.expanduser()
public_path = args.public_key.expanduser() if args.public_key else None
keyring_path = args.keyring.expanduser() if args.keyring else None
if private_path.exists() and not args.force:
raise SystemExit(f"Private key already exists: {private_path}")
if public_path is not None and public_path.exists() and not args.force:
raise SystemExit(f"Public key already exists: {public_path}")
private_key = Ed25519PrivateKey.generate()
private_bytes = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
)
public_bytes = private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
public_base64 = base64.b64encode(public_bytes).decode("ascii")
private_path.parent.mkdir(parents=True, exist_ok=True)
private_path.write_bytes(private_bytes)
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
if public_path is not None:
public_path.parent.mkdir(parents=True, exist_ok=True)
public_path.write_text(public_base64 + "\n", encoding="utf-8")
if keyring_path is not None:
keyring_path.parent.mkdir(parents=True, exist_ok=True)
keyring = {
"keyring_version": "1",
"purpose": "govoplan module package catalog signatures",
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
"keys": [
{
"key_id": args.key_id,
"status": args.status,
"public_key": public_base64,
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
}
],
}
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"private_key={private_path}")
if public_path is not None:
print(f"public_key={public_path}")
if keyring_path is not None:
print(f"keyring={keyring_path}")
print(f"key_id={args.key_id}")
print(f"public_key_base64={public_base64}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

408
tools/release/generate-release-catalog.py Normal file
View File

@@ -0,0 +1,408 @@
#!/usr/bin/env python3
"""Generate and sign a GovOPlaN module package release catalog."""
from __future__ import annotations
import argparse
import base64
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import json
import os
from pathlib import Path
import sys
from typing import Any
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
META_ROOT = Path(__file__).resolve().parents[2]
CORE_ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
sys.path.insert(0, str(CORE_ROOT / "src"))
from govoplan_core.core.modules import ModuleManifest # noqa: E402
from govoplan_core.server.registry import available_module_manifests # noqa: E402
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
@dataclass(frozen=True, slots=True)
class CatalogModule:
module_id: str
repo: str
python_package: str
name: str
description: str
tags: tuple[str, ...]
webui_package: str | None = None
provides_interfaces: tuple[dict[str, object], ...] = ()
requires_interfaces: tuple[dict[str, object], ...] = ()
CATALOG_MODULES = (
CatalogModule(
module_id="tenancy",
repo="govoplan-tenancy",
python_package="govoplan-tenancy",
name="Tenancy",
description="Tenant registry, tenant settings, and tenant resolution platform module.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="organizations",
repo="govoplan-organizations",
python_package="govoplan-organizations",
name="Organizations",
description="Organization units, functions, and account-held function assignments.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="identity",
repo="govoplan-identity",
python_package="govoplan-identity",
name="Identity",
description="Canonical identities and links between identities and platform accounts.",
tags=("official", "platform-module"),
),
CatalogModule(
module_id="access",
repo="govoplan-access",
python_package="govoplan-access",
name="Access",
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
tags=("official", "platform-module"),
webui_package="@govoplan/access-webui",
),
CatalogModule(
module_id="admin",
repo="govoplan-admin",
python_package="govoplan-admin",
name="Admin",
description="System settings, governance templates, module management, and admin shell contributions.",
tags=("official", "platform-module"),
webui_package="@govoplan/admin-webui",
),
CatalogModule(
module_id="policy",
repo="govoplan-policy",
python_package="govoplan-policy",
name="Policy",
description="Policy and governance capability module.",
tags=("official", "platform-module"),
webui_package="@govoplan/policy-webui",
),
CatalogModule(
module_id="audit",
repo="govoplan-audit",
python_package="govoplan-audit",
name="Audit",
description="Audit-log storage and audit administration routes.",
tags=("official", "platform-module"),
webui_package="@govoplan/audit-webui",
),
CatalogModule(
module_id="dashboard",
repo="govoplan-dashboard",
python_package="govoplan-dashboard",
name="Dashboard",
description="Configurable user home assembled from module-provided dashboard widgets.",
tags=("official", "platform-module"),
webui_package="@govoplan/dashboard-webui",
),
CatalogModule(
module_id="files",
repo="govoplan-files",
python_package="govoplan-files",
name="Files",
description="Managed file spaces and campaign attachment integration.",
tags=("official", "service-module"),
webui_package="@govoplan/files-webui",
),
CatalogModule(
module_id="mail",
repo="govoplan-mail",
python_package="govoplan-mail",
name="Mail",
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
tags=("official", "service-module"),
webui_package="@govoplan/mail-webui",
),
CatalogModule(
module_id="campaigns",
repo="govoplan-campaign",
python_package="govoplan-campaign",
name="Campaigns",
description="Campaign authoring, validation, queueing, delivery control, and reports.",
tags=("official", "business-module"),
webui_package="@govoplan/campaign-webui",
),
CatalogModule(
module_id="calendar",
repo="govoplan-calendar",
python_package="govoplan-calendar",
name="Calendar",
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
tags=("official", "service-module"),
webui_package="@govoplan/calendar-webui",
),
CatalogModule(
module_id="docs",
repo="govoplan-docs",
python_package="govoplan-docs",
name="Docs",
description="Configured-system documentation and evidence-aware help surfaces.",
tags=("official", "platform-module"),
webui_package="@govoplan/docs-webui",
),
CatalogModule(
module_id="ops",
repo="govoplan-ops",
python_package="govoplan-ops",
name="Ops",
description="Runtime health, deployment profile, worker split, and sizing visibility.",
tags=("official", "platform-module"),
webui_package="@govoplan/ops-webui",
),
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
parser.add_argument("--channel", default="stable")
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
parser.add_argument("--expires-days", type=int, default=90)
parser.add_argument("--catalog-output", type=Path, required=True)
parser.add_argument("--keyring-output", type=Path)
parser.add_argument(
"--catalog-signing-key",
action="append",
default=[],
metavar="KEY_ID=PRIVATE_KEY",
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
)
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
parser.add_argument("--repository-base", default=GITEA_BASE)
args = parser.parse_args()
version = args.version.removeprefix("v")
tag = f"v{version}"
generated_at = datetime.now(tz=UTC)
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
expires_at = generated_at + timedelta(days=args.expires_days)
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
catalog = _catalog_payload(
version=version,
tag=tag,
channel=args.channel,
sequence=sequence,
generated_at=generated_at,
expires_at=expires_at,
repository_base=args.repository_base.rstrip("/"),
public_base_url=args.public_base_url.rstrip("/"),
)
if signing_keys:
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
output = args.catalog_output.expanduser()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.keyring_output is not None:
keyring_output = args.keyring_output.expanduser()
keyring_output.parent.mkdir(parents=True, exist_ok=True)
keyring_output.write_text(
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
print(f"catalog={output}")
if args.keyring_output is not None:
print(f"keyring={args.keyring_output.expanduser()}")
print(f"channel={args.channel}")
print(f"sequence={sequence}")
print(f"version={version}")
return 0
def _catalog_payload(
*,
version: str,
tag: str,
channel: str,
sequence: int,
generated_at: datetime,
expires_at: datetime,
repository_base: str,
public_base_url: str,
) -> dict[str, Any]:
manifests = _discovered_catalog_manifests()
modules: list[dict[str, Any]] = []
for module in CATALOG_MODULES:
manifest = manifests.get(module.module_id)
module_version = manifest.version if manifest is not None else version
module_tag = f"v{module_version.removeprefix('v')}"
entry: dict[str, Any] = {
"module_id": module.module_id,
"name": module.name,
"description": module.description,
"version": module_version,
"action": "install",
"python_package": module.python_package,
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
"license_features": [f"module.{module.module_id}"],
"tags": list(module.tags),
}
if module.webui_package:
entry["webui_package"] = module.webui_package
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
manifest_metadata = _manifest_catalog_metadata(manifest)
entry.update(manifest_metadata)
if module.provides_interfaces:
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
if module.requires_interfaces:
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
modules.append(entry)
return {
"catalog_version": "1",
"channel": channel,
"sequence": sequence,
"generated_at": _json_datetime(generated_at),
"expires_at": _json_datetime(expires_at),
"release": {
"version": version,
"tag": tag,
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
},
"core_release": {
"name": "GovOPlaN Core",
"version": version,
"python_package": "govoplan-core",
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
"webui_package": "@govoplan/core-webui",
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
},
"modules": modules,
}
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
try:
return available_module_manifests(ignore_load_errors=True)
except Exception:
return {}
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
if manifest is None:
return {}
payload: dict[str, object] = {}
if manifest.dependencies:
payload["dependencies"] = list(manifest.dependencies)
if manifest.optional_dependencies:
payload["optional_dependencies"] = list(manifest.optional_dependencies)
if manifest.migration_spec is not None:
payload["migration_safety"] = "requires_review"
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
if manifest.migration_spec.migration_after:
payload["migration_after"] = list(manifest.migration_spec.migration_after)
if manifest.migration_spec.migration_before:
payload["migration_before"] = list(manifest.migration_spec.migration_before)
if manifest.migration_spec.migration_tasks:
tasks: list[dict[str, object]] = []
for task in manifest.migration_spec.migration_tasks:
task_payload: dict[str, object] = {
"task_id": task.task_id,
"phase": task.phase,
"summary": task.summary,
"task_version": task.task_version,
"safety": task.safety,
"idempotent": task.idempotent,
}
if task.timeout_seconds is not None:
task_payload["timeout_seconds"] = task.timeout_seconds
tasks.append(task_payload)
payload["migration_tasks"] = tasks
if manifest.provides_interfaces:
payload["provides_interfaces"] = [
{"name": item.name, "version": item.version}
for item in manifest.provides_interfaces
]
if manifest.requires_interfaces:
requirements: list[dict[str, object]] = []
for item in manifest.requires_interfaces:
requirement: dict[str, object] = {
"name": item.name,
"optional": item.optional,
}
if item.version_min is not None:
requirement["version_min"] = item.version_min
if item.version_max_exclusive is not None:
requirement["version_max_exclusive"] = item.version_max_exclusive
requirements.append(requirement)
payload["requires_interfaces"] = requirements
return payload
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
key_id, separator, path_text = value.partition("=")
if not separator or not key_id.strip() or not path_text.strip():
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
path = Path(path_text).expanduser()
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
if not isinstance(private_key, Ed25519PrivateKey):
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
return key_id.strip(), private_key
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
signature_payload = dict(payload)
signature_payload.pop("signature", None)
signature_payload.pop("signatures", None)
signature = private_key.sign(_canonical_bytes(signature_payload))
return {
"algorithm": "ed25519",
"key_id": key_id,
"value": base64.b64encode(signature).decode("ascii"),
}
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
return {
"keyring_version": "1",
"purpose": "govoplan module package catalog signatures",
"generated_at": _json_datetime(generated_at),
"keys": [
{
"key_id": key_id,
"status": "active",
"public_key": _public_key_base64(private_key),
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
}
for key_id, private_key in signing_keys
],
}
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
public_bytes = private_key.public_key().public_bytes(
encoding=serialization.Encoding.Raw,
format=serialization.PublicFormat.Raw,
)
return base64.b64encode(public_bytes).decode("ascii")
def _canonical_bytes(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def _json_datetime(value: datetime) -> str:
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
if __name__ == "__main__":
raise SystemExit(main())

116
tools/release/generate-release-lock.sh Normal file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'USAGE'
Usage:
tools/release/generate-release-lock.sh [options]
Generates webui/package-lock.release.json from webui/package.release.json in a
temporary workspace. The normal development package.json and package-lock.json
are left untouched.
Run this after the module git tags referenced by package.release.json exist and
are reachable, and before tagging the core release commit that should contain
the regenerated release lockfile.
Options:
--npm <path> npm executable to use.
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
-h, --help Show this help.
USAGE
}
fail() {
echo "error: $*" >&2
exit 1
}
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CORE_ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
CORE_ROOT="$(cd "$CORE_ROOT" && pwd)"
WEBUI="$CORE_ROOT/webui"
NPM_BIN="${NPM:-}"
NODE_BIN="${NODE:-}"
if [[ -z "$NPM_BIN" ]]; then
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
else
NPM_BIN="npm"
fi
fi
if [[ -z "$NODE_BIN" ]]; then
if [[ -x "$(dirname "$NPM_BIN")/node" ]]; then
NODE_BIN="$(dirname "$NPM_BIN")/node"
else
NODE_BIN="node"
fi
fi
while [[ $# -gt 0 ]]; do
case "$1" in
--npm)
[[ $# -ge 2 ]] || fail "missing value for $1"
NPM_BIN="$2"
shift 2
;;
--core-root)
[[ $# -ge 2 ]] || fail "missing value for $1"
CORE_ROOT="$(cd "$2" && pwd)"
WEBUI="$CORE_ROOT/webui"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
[[ -f "$WEBUI/package.release.json" ]] || fail "missing $WEBUI/package.release.json"
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm executable not found: $NPM_BIN"
command -v "$NODE_BIN" >/dev/null 2>&1 || fail "node executable not found: $NODE_BIN"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-lock.XXXXXXXX")"
cleanup() {
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"
if [[ -f "$WEBUI/package-lock.release.json" ]]; then
cp "$WEBUI/package-lock.release.json" "$TMP_DIR/package-lock.json"
fi
echo "Generating release lockfile from $WEBUI/package.release.json"
echo "Temporary workspace: $TMP_DIR"
(
cd "$TMP_DIR"
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
mapfile -t GIT_PACKAGES < <(
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
const fs = require("fs");
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
for (const group of ["dependencies", "devDependencies", "optionalDependencies"]) {
for (const [name, spec] of Object.entries(pkg[group] || {})) {
if (typeof spec === "string" && spec.startsWith("git+")) {
console.log(name);
}
}
}
NODE
)
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
fi
)
cp "$TMP_DIR/package-lock.json" "$WEBUI/package-lock.release.json"
echo "Updated $WEBUI/package-lock.release.json"

76
tools/release/install-webui-release-dependencies.sh Normal file
View File

@@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -euo pipefail
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CORE_ROOT="${GOVOPLAN_CORE_ROOT:-$META_ROOT/../govoplan-core}"
CORE_ROOT="$(cd "$CORE_ROOT" && pwd)"
WEBUI_DIR="${1:-$CORE_ROOT/webui}"
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
export GOVOPLAN_DEPS
trap 'rm -rf "$WORK_ROOT"' EXIT
retry() {
local attempt status
for attempt in 1 2 3; do
if "$@"; then
return 0
fi
status=$?
if [[ "$attempt" == 3 ]]; then
return "$status"
fi
sleep $((attempt * 10))
done
}
cd "$WEBUI_DIR"
node <<'JS'
const fs = require("fs");
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
const rel = JSON.parse(fs.readFileSync("package.release.json", "utf8"));
const govoplanDeps = [];
const dependencies = {...(rel.dependencies || {})};
for (const [name, spec] of Object.entries(dependencies)) {
if (name.startsWith("@govoplan/")) {
govoplanDeps.push([name, spec]);
delete dependencies[name];
}
}
for (const key of ["devDependencies", "peerDependencies", "optionalDependencies", "overrides"]) {
if (rel[key]) pkg[key] = rel[key];
}
pkg.dependencies = dependencies;
fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
fs.writeFileSync(process.env.GOVOPLAN_DEPS, govoplanDeps.map((item) => item.join("\t")).join("\n") + "\n");
JS
rm -f package-lock.json
npm cache clean --force
retry npm install --prefer-online
module_paths=()
while IFS=$'\t' read -r package_name spec; do
[[ -n "${package_name:-}" ]] || continue
git_url="${spec%%#*}"
git_ref="${spec#*#}"
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
echo "Unsupported GovOPlaN WebUI git spec for $package_name: $spec" >&2
exit 1
fi
clone_url="${git_url#git+}"
clone_dir="$WORK_ROOT/${package_name#@govoplan/}"
echo "Installing $package_name from $clone_url#$git_ref"
retry git clone --depth 1 --branch "$git_ref" "$clone_url" "$clone_dir"
module_paths+=("file:$clone_dir")
done < "$GOVOPLAN_DEPS"
if [[ "${#module_paths[@]}" -gt 0 ]]; then
retry npm install --prefer-online --no-save --install-links "${module_paths[@]}"
fi

260
tools/release/publish-release-catalog.sh Normal file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'USAGE'
Usage:
tools/release/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
Generates a signed module package catalog for a GovOPlaN release, writes it into
addideas-govoplan-website/public/catalogs/v1, validates it, optionally builds the website,
and optionally commits/tags/pushes addideas-govoplan-website.
Options:
--version <x.y.z> Release version without leading v.
--channel <name> Catalog channel. Defaults to stable.
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
--expires-days <days> Catalog expiry window. Defaults to 90.
--catalog-signing-key <key-id=/path/private.pem>
Ed25519 private key. May be repeated for rotation.
--core-root <path> govoplan-core checkout. Defaults to ../govoplan-core.
--web-root <path> addideas-govoplan-website checkout. Defaults to ../addideas-govoplan-website.
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
--npm <path> npm executable used for optional web build.
--build-web Run npm run build in addideas-govoplan-website after generating files.
--commit Commit generated catalog files in addideas-govoplan-website.
--tag Tag addideas-govoplan-website with catalog-v<x.y.z>. Implies --commit.
--push Push addideas-govoplan-website branch and tag. Implies --commit.
--remote <name> Git remote for push. Defaults to origin.
--branch <name> Branch to push. Defaults to current addideas-govoplan-website branch.
-n, --dry-run Print commands and validate inputs without writing git changes.
-h, --help Show this help.
USAGE
}
fail() {
echo "error: $*" >&2
exit 1
}
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
PARENT="$(dirname "$META_ROOT")"
CORE_ROOT="${GOVOPLAN_CORE_ROOT:-$PARENT/govoplan-core}"
CORE_ROOT="$(cd "$CORE_ROOT" && pwd)"
WEB_ROOT="$PARENT/addideas-govoplan-website"
VERSION=""
CHANNEL="stable"
SEQUENCE=""
EXPIRES_DAYS="90"
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
REMOTE="origin"
BRANCH=""
BUILD_WEB=0
COMMIT=0
TAG_WEB=0
PUSH=0
DRY_RUN=0
SIGNING_KEYS=()
NPM_BIN="${NPM:-}"
if [[ -z "${PYTHON:-}" ]]; then
if [[ -x "$META_ROOT/.venv/bin/python" ]]; then
PYTHON="$META_ROOT/.venv/bin/python"
else
PYTHON="python3"
fi
fi
if [[ -z "$NPM_BIN" ]]; then
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
else
NPM_BIN="npm"
fi
fi
while [[ $# -gt 0 ]]; do
case "$1" in
--version)
[[ $# -ge 2 ]] || fail "missing value for $1"
VERSION="${2#v}"
shift 2
;;
--channel)
[[ $# -ge 2 ]] || fail "missing value for $1"
CHANNEL="$2"
shift 2
;;
--sequence)
[[ $# -ge 2 ]] || fail "missing value for $1"
SEQUENCE="$2"
shift 2
;;
--expires-days)
[[ $# -ge 2 ]] || fail "missing value for $1"
EXPIRES_DAYS="$2"
shift 2
;;
--catalog-signing-key)
[[ $# -ge 2 ]] || fail "missing value for $1"
SIGNING_KEYS+=("$2")
shift 2
;;
--core-root)
[[ $# -ge 2 ]] || fail "missing value for $1"
CORE_ROOT="$(cd "$2" && pwd)"
shift 2
;;
--web-root)
[[ $# -ge 2 ]] || fail "missing value for $1"
WEB_ROOT="$(cd "$2" && pwd)"
shift 2
;;
--public-base-url)
[[ $# -ge 2 ]] || fail "missing value for $1"
PUBLIC_BASE_URL="$2"
shift 2
;;
--npm)
[[ $# -ge 2 ]] || fail "missing value for $1"
NPM_BIN="$2"
shift 2
;;
--build-web)
BUILD_WEB=1
shift
;;
--commit)
COMMIT=1
shift
;;
--tag)
TAG_WEB=1
COMMIT=1
shift
;;
--push)
PUSH=1
COMMIT=1
shift
;;
--remote)
[[ $# -ge 2 ]] || fail "missing value for $1"
REMOTE="$2"
shift 2
;;
--branch)
[[ $# -ge 2 ]] || fail "missing value for $1"
BRANCH="$2"
shift 2
;;
-n|--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
[[ -n "$VERSION" ]] || fail "--version is required"
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
[[ -d "$WEB_ROOT/.git" ]] || fail "not an addideas-govoplan-website git repo: $WEB_ROOT"
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
if [[ "$BUILD_WEB" -eq 1 ]]; then
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
fi
if [[ -z "$BRANCH" ]]; then
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
fi
[[ -n "$BRANCH" ]] || fail "cannot infer addideas-govoplan-website branch; pass --branch"
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
TAG_NAME="catalog-v$VERSION"
run() {
printf '+'
printf ' %q' "$@"
printf '\n'
if [[ "$DRY_RUN" -eq 0 ]]; then
"$@"
fi
}
GEN_ARGS=(
"$PYTHON" "$META_ROOT/tools/release/generate-release-catalog.py"
--version "$VERSION"
--channel "$CHANNEL"
--expires-days "$EXPIRES_DAYS"
--catalog-output "$CATALOG_PATH"
--keyring-output "$KEYRING_PATH"
--public-base-url "$PUBLIC_BASE_URL"
)
if [[ -n "$SEQUENCE" ]]; then
GEN_ARGS+=(--sequence "$SEQUENCE")
fi
for signing_key in "${SIGNING_KEYS[@]}"; do
GEN_ARGS+=(--catalog-signing-key "$signing_key")
done
run "${GEN_ARGS[@]}"
if [[ "$DRY_RUN" -eq 0 ]]; then
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
"$PYTHON" -m govoplan_core.commands.module_installer \
--validate-package-catalog "$CATALOG_PATH" \
--require-signed-catalog \
--approved-catalog-channel "$CHANNEL" \
--format json >/dev/null
else
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
fi
if [[ "$BUILD_WEB" -eq 1 ]]; then
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
fi
if [[ "$COMMIT" -eq 1 ]]; then
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
if [[ "$DRY_RUN" -eq 0 ]]; then
if git -C "$WEB_ROOT" diff --cached --quiet; then
echo "No addideas-govoplan-website catalog changes to commit."
else
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
fi
else
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
fi
fi
if [[ "$TAG_WEB" -eq 1 ]]; then
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
fail "addideas-govoplan-website tag already exists: $TAG_NAME"
fi
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
fi
if [[ "$PUSH" -eq 1 ]]; then
if [[ "$TAG_WEB" -eq 1 ]]; then
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
else
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
fi
fi
echo "Catalog ready:"
echo " $CATALOG_PATH"
echo " $KEYRING_PATH"
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"

864
tools/release/push-release-tag.sh Normal file
View File

@@ -0,0 +1,864 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'USAGE'
Usage:
tools/release/push-release-tag.sh [options]
Bumps installable GovOPlaN package versions, commits pending changes in the
GovOPlaN release/module repositories, creates an annotated release tag in each
repo, and pushes branch+tag.
By default the script asks which version part to bump:
major X.Y.Z -> (X+1).0.0
minor X.Y.Z -> X.(Y+1).0
subversion X.Y.Z -> X.Y.(Z+1)
Options:
--bump <kind> Version bump: major, minor, subversion, or patch.
--version <x.y.z> Explicit target version instead of a bump. May equal
the current repo versions when they were bumped already.
-r, --remote <name> Remote to push to. Defaults to origin.
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
--tag-message <text> Annotated tag message. Defaults to the commit message.
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
--warn-migration-audit Run the migration release audit without baseline strictness.
--strict-migration-audit
Require current Alembic heads to be recorded in the
latest migration release baseline.
--skip-migration-audit Do not run the release migration audit.
--publish-web-catalog After core/modules are pushed, publish a signed release
catalog into addideas-govoplan-website.
--web-root <path> addideas-govoplan-website checkout for --publish-web-catalog.
Defaults to ../addideas-govoplan-website.
--catalog-signing-key <key-id=/path/private.pem>
Catalog signing key for --publish-web-catalog. May be
repeated during key rotation.
--catalog-channel <name>
Catalog channel. Defaults to stable.
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
--catalog-expires-days <days>
Catalog expiry window. Defaults to 90.
--catalog-public-url <url>
Public website URL. Defaults to
https://govoplan.add-ideas.de.
--build-web-catalog Run npm run build in addideas-govoplan-website before committing the
catalog update.
--npm <path> npm executable for lockfile generation.
-y, --yes Do not prompt before committing, tagging, and pushing.
-n, --dry-run Print intended actions without changing files or git state.
-h, --help Show this help.
Repos:
Installable release packages:
govoplan-access
govoplan-admin
govoplan-tenancy
govoplan-organizations
govoplan-identity
govoplan-idm
govoplan-policy
govoplan-audit
govoplan-dashboard
govoplan-files
govoplan-mail
govoplan-campaign
govoplan-calendar
govoplan-ops
govoplan-core
Roadmap/scaffold module repositories are tag-only until they have package
metadata: addresses, appointments, cases, connectors, dms, erp,
fit-connect, forms, identity-trust, ledger, notifications, payments,
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
xrechnung, and xta-osci.
Examples:
tools/release/push-release-tag.sh
tools/release/push-release-tag.sh --bump subversion
tools/release/push-release-tag.sh --version 0.2.0 --message "Release v0.2.0"
USAGE
}
fail() {
echo "error: $*" >&2
exit 1
}
META_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
PARENT="$(dirname "$META_ROOT")"
ROOT="${GOVOPLAN_CORE_ROOT:-$PARENT/govoplan-core}"
ROOT="$(cd "$ROOT" && pwd)"
PACKAGE_MODULE_REPOS=(
"$PARENT/govoplan-access"
"$PARENT/govoplan-admin"
"$PARENT/govoplan-approvals"
"$PARENT/govoplan-assets"
"$PARENT/govoplan-audit"
"$PARENT/govoplan-booking"
"$PARENT/govoplan-calendar"
"$PARENT/govoplan-campaign"
"$PARENT/govoplan-certificates"
"$PARENT/govoplan-committee"
"$PARENT/govoplan-consultation"
"$PARENT/govoplan-contracts"
"$PARENT/govoplan-dashboard"
"$PARENT/govoplan-docs"
"$PARENT/govoplan-facilities"
"$PARENT/govoplan-files"
"$PARENT/govoplan-forms-runtime"
"$PARENT/govoplan-grants"
"$PARENT/govoplan-helpdesk"
"$PARENT/govoplan-identity"
"$PARENT/govoplan-idm"
"$PARENT/govoplan-inspections"
"$PARENT/govoplan-issue-reporting"
"$PARENT/govoplan-learning"
"$PARENT/govoplan-mail"
"$PARENT/govoplan-ops"
"$PARENT/govoplan-organizations"
"$PARENT/govoplan-permits"
"$PARENT/govoplan-policy"
"$PARENT/govoplan-procurement"
"$PARENT/govoplan-records"
"$PARENT/govoplan-resources"
"$PARENT/govoplan-risk-compliance"
"$PARENT/govoplan-tenancy"
"$PARENT/govoplan-transparency"
)
TAG_ONLY_MODULE_REPOS=(
"$PARENT/govoplan-addresses"
"$PARENT/govoplan-appointments"
"$PARENT/govoplan-cases"
"$PARENT/govoplan-connectors"
"$PARENT/govoplan-dms"
"$PARENT/govoplan-erp"
"$PARENT/govoplan-fit-connect"
"$PARENT/govoplan-forms"
"$PARENT/govoplan-identity-trust"
"$PARENT/govoplan-ledger"
"$PARENT/govoplan-notifications"
"$PARENT/govoplan-payments"
"$PARENT/govoplan-portal"
"$PARENT/govoplan-postbox"
"$PARENT/govoplan-reporting"
"$PARENT/govoplan-scheduling"
"$PARENT/govoplan-search"
"$PARENT/govoplan-tasks"
"$PARENT/govoplan-templates"
"$PARENT/govoplan-workflow"
"$PARENT/govoplan-xoev"
"$PARENT/govoplan-xrechnung"
"$PARENT/govoplan-xta-osci"
)
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
REPOS=(
"${MODULE_REPOS[@]}"
"$ROOT"
)
REMOTE="origin"
BRANCH=""
BUMP=""
TARGET_VERSION=""
COMMIT_MESSAGE=""
TAG_MESSAGE=""
DRY_RUN=0
YES=0
GENERATE_RELEASE_LOCK=1
MIGRATION_AUDIT_MODE="auto"
NPM_BIN="${NPM:-}"
PUBLISH_WEB_CATALOG=0
WEB_ROOT="$PARENT/addideas-govoplan-website"
CATALOG_CHANNEL="stable"
CATALOG_SEQUENCE=""
CATALOG_EXPIRES_DAYS="90"
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
BUILD_WEB_CATALOG=0
CATALOG_SIGNING_KEYS=()
if [[ -z "${PYTHON:-}" ]]; then
if [[ -x "$META_ROOT/.venv/bin/python" ]]; then
PYTHON="$META_ROOT/.venv/bin/python"
else
PYTHON="python3"
fi
fi
if [[ -z "$NPM_BIN" ]]; then
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
else
NPM_BIN="npm"
fi
fi
export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o BatchMode=yes -o ConnectTimeout=15}"
GIT_REMOTE_TIMEOUT="${GIT_REMOTE_TIMEOUT:-30}"
normalize_bump() {
local value="${1,,}"
case "$value" in
major|minor|subversion)
printf '%s\n' "$value"
;;
patch|sub|sub-version)
printf '%s\n' "subversion"
;;
*)
return 1
;;
esac
}
validate_version() {
[[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]
}
version_gt() {
local new_major new_minor new_patch old_major old_minor old_patch
IFS=. read -r new_major new_minor new_patch <<< "$1"
IFS=. read -r old_major old_minor old_patch <<< "$2"
(( new_major > old_major )) && return 0
(( new_major < old_major )) && return 1
(( new_minor > old_minor )) && return 0
(( new_minor < old_minor )) && return 1
(( new_patch > old_patch )) && return 0
return 1
}
bump_version() {
local version="$1"
local bump="$2"
local major minor patch
IFS=. read -r major minor patch <<< "$version"
case "$bump" in
major)
printf '%s.0.0\n' "$((major + 1))"
;;
minor)
printf '%s.%s.0\n' "$major" "$((minor + 1))"
;;
subversion)
printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))"
;;
*)
fail "unknown bump kind: $bump"
;;
esac
}
get_project_name() {
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["name"])' "$1/pyproject.toml"
}
get_project_version() {
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$1/pyproject.toml"
}
update_pyproject() {
local repo="$1"
local version="$2"
"$PYTHON" - "$repo/pyproject.toml" "$version" <<'PYCODE'
from __future__ import annotations
import pathlib
import re
import sys
import tomllib
path = pathlib.Path(sys.argv[1])
new_version = sys.argv[2]
text = path.read_text()
data = tomllib.loads(text)
project = data["project"]
name = project["name"]
old_version = project["version"]
text, version_count = re.subn(
r'(?m)^version\s*=\s*["\'][^"\']+["\']\s*$',
f'version = "{new_version}"',
text,
count=1,
)
if version_count != 1:
raise SystemExit(f"could not update project.version in {path}")
if name != "govoplan-core":
text, dependency_count = re.subn(
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
rf'\g<1>{new_version}',
text,
)
if dependency_count < 1:
raise SystemExit(f"could not update govoplan-core dependency in {path}")
path.write_text(text)
print(f"{name}: {old_version} -> {new_version}")
PYCODE
}
update_manifest_version() {
local repo="$1"
local project_name="$2"
local version="$3"
local manifest_paths=()
if [[ "$project_name" == "govoplan-core" ]]; then
return 0
fi
while IFS= read -r -d '' manifest_path; do
manifest_paths+=("$manifest_path")
done < <(find "$repo/src" -path "*/backend/manifest.py" -print0 2>/dev/null)
[[ "${#manifest_paths[@]}" -gt 0 ]] || fail "missing module manifest under $repo/src"
for manifest_path in "${manifest_paths[@]}"; do
"$PYTHON" - "$manifest_path" "$version" <<'PYCODE'
from __future__ import annotations
import pathlib
import re
import sys
path = pathlib.Path(sys.argv[1])
new_version = sys.argv[2]
text = path.read_text()
text, count = re.subn(
r'(?m)^(\s*version=)["\'][^"\']+["\'](,?\s*)$',
rf'\1"{new_version}"\2',
text,
count=1,
)
if count == 0:
text, count = re.subn(
r'(?m)^(MODULE_VERSION\s*=\s*)["\'][^"\']+["\'](\s*)$',
rf'\1"{new_version}"\2',
text,
count=1,
)
if count != 1:
raise SystemExit(f"could not update ModuleManifest.version in {path}")
path.write_text(text)
PYCODE
done
}
update_webui_package() {
local repo="$1"
local project_name="$2"
local version="$3"
local package_path=""
local release_package_path="$repo/webui/package.release.json"
for package_path in "$repo/package.json" "$repo/webui/package.json"; do
[[ -f "$package_path" ]] || continue
"$PYTHON" - "$package_path" "$project_name" "$version" <<'PYCODE'
from __future__ import annotations
import json
import pathlib
import sys
path = pathlib.Path(sys.argv[1])
project_name = sys.argv[2]
new_version = sys.argv[3]
data = json.loads(path.read_text())
data["version"] = new_version
if project_name != "govoplan-core":
peers = data.setdefault("peerDependencies", {})
if "@govoplan/core-webui" in peers:
peers["@govoplan/core-webui"] = f"^{new_version}"
path.write_text(json.dumps(data, indent=2) + "\n")
PYCODE
done
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
"$PYTHON" - "$release_package_path" "$version" <<'PYCODE'
from __future__ import annotations
import json
import pathlib
import re
import sys
path = pathlib.Path(sys.argv[1])
new_version = sys.argv[2]
data = json.loads(path.read_text())
data["version"] = new_version
dependencies = data.setdefault("dependencies", {})
for name, spec in list(dependencies.items()):
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
path.write_text(json.dumps(data, indent=2) + "\n")
PYCODE
fi
}
update_version_files() {
local repo="$1"
local version="$2"
local project_name="${PROJECT_NAMES[$repo]}"
update_pyproject "$repo" "$version"
update_manifest_version "$repo" "$project_name" "$version"
update_webui_package "$repo" "$project_name" "$version"
}
refresh_development_webui_lock() {
[[ -f "$ROOT/webui/package.json" ]] || return 0
[[ -f "$ROOT/webui/package-lock.json" ]] || return 0
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$ROOT/webui" install --package-lock-only --ignore-scripts
}
generate_release_lock() {
if [[ "$GENERATE_RELEASE_LOCK" -eq 0 ]]; then
return
fi
[[ -x "$META_ROOT/tools/release/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $META_ROOT/tools/release/generate-release-lock.sh"
run "$META_ROOT/tools/release/generate-release-lock.sh" --core-root "$ROOT" --npm "$NPM_BIN"
}
run_migration_release_audit() {
local audit_script="$META_ROOT/tools/release/release-migration-audit.py"
local command=("$PYTHON" "$audit_script" "--track" "release")
case "$MIGRATION_AUDIT_MODE" in
skip)
echo "Migration release audit: skipped"
return
;;
strict)
command+=("--strict")
;;
auto)
command+=("--strict-if-baseline")
;;
warn)
;;
*)
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
;;
esac
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
run "${command[@]}"
}
print_command() {
printf '+'
printf ' %q' "$@"
printf '\n'
}
run() {
print_command "$@"
if [[ "$DRY_RUN" -eq 0 ]]; then
"$@"
fi
}
prompt_for_bump() {
local answer=""
if [[ -n "$BUMP" || -n "$TARGET_VERSION" ]]; then
return
fi
if [[ ! -t 0 ]]; then
fail "pass --bump major|minor|subversion or --version x.y.z for non-interactive use"
fi
echo "Select version increase:"
echo " 1) major X.Y.Z -> (X+1).0.0"
echo " 2) minor X.Y.Z -> X.(Y+1).0"
echo " 3) subversion X.Y.Z -> X.Y.(Z+1)"
printf 'Choice [subversion]: '
read -r answer
case "${answer,,}" in
""|3|subversion|patch|sub|sub-version)
BUMP="subversion"
;;
1|major)
BUMP="major"
;;
2|minor)
BUMP="minor"
;;
*)
fail "unknown version bump: $answer"
;;
esac
}
confirm_release() {
local answer=""
if [[ "$DRY_RUN" -eq 1 || "$YES" -eq 1 ]]; then
return
fi
if [[ ! -t 0 ]]; then
fail "refusing to commit, tag, and push without confirmation; pass --yes for non-interactive use"
fi
printf 'Commit all changes, tag all repos, and push to %s? [y/N] ' "$REMOTE"
read -r answer
case "${answer,,}" in
y|yes)
;;
*)
echo "Aborted."
exit 1
;;
esac
}
while [[ $# -gt 0 ]]; do
case "$1" in
--bump)
[[ $# -ge 2 ]] || fail "missing value for $1"
BUMP="$(normalize_bump "$2")" || fail "unknown bump kind: $2"
shift 2
;;
--version)
[[ $# -ge 2 ]] || fail "missing value for $1"
TARGET_VERSION="$2"
shift 2
;;
-r|--remote)
[[ $# -ge 2 ]] || fail "missing value for $1"
REMOTE="$2"
shift 2
;;
-b|--branch)
[[ $# -ge 2 ]] || fail "missing value for $1"
BRANCH="$2"
shift 2
;;
-m|--message)
[[ $# -ge 2 ]] || fail "missing value for $1"
COMMIT_MESSAGE="$2"
shift 2
;;
--tag-message)
[[ $# -ge 2 ]] || fail "missing value for $1"
TAG_MESSAGE="$2"
shift 2
;;
--skip-release-lock)
GENERATE_RELEASE_LOCK=0
shift
;;
--warn-migration-audit)
MIGRATION_AUDIT_MODE="warn"
shift
;;
--strict-migration-audit)
MIGRATION_AUDIT_MODE="strict"
shift
;;
--skip-migration-audit)
MIGRATION_AUDIT_MODE="skip"
shift
;;
--publish-web-catalog)
PUBLISH_WEB_CATALOG=1
shift
;;
--web-root)
[[ $# -ge 2 ]] || fail "missing value for $1"
WEB_ROOT="$2"
shift 2
;;
--catalog-signing-key)
[[ $# -ge 2 ]] || fail "missing value for $1"
CATALOG_SIGNING_KEYS+=("$2")
shift 2
;;
--catalog-channel)
[[ $# -ge 2 ]] || fail "missing value for $1"
CATALOG_CHANNEL="$2"
shift 2
;;
--catalog-sequence)
[[ $# -ge 2 ]] || fail "missing value for $1"
CATALOG_SEQUENCE="$2"
shift 2
;;
--catalog-expires-days)
[[ $# -ge 2 ]] || fail "missing value for $1"
CATALOG_EXPIRES_DAYS="$2"
shift 2
;;
--catalog-public-url)
[[ $# -ge 2 ]] || fail "missing value for $1"
CATALOG_PUBLIC_URL="$2"
shift 2
;;
--build-web-catalog)
BUILD_WEB_CATALOG=1
shift
;;
--npm)
[[ $# -ge 2 ]] || fail "missing value for $1"
NPM_BIN="$2"
shift 2
;;
-y|--yes)
YES=1
shift
;;
-n|--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
fail "use either --bump or --version, not both"
fi
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
fail "--publish-web-catalog requires at least one --catalog-signing-key"
fi
prompt_for_bump
if ! command -v "$PYTHON" >/dev/null 2>&1; then
fail "Python interpreter not found: $PYTHON"
fi
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
fail "npm executable not found: $NPM_BIN"
fi
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
[[ -d "$WEB_ROOT/.git" ]] || fail "addideas-govoplan-website repo not found: $WEB_ROOT"
[[ -x "$META_ROOT/tools/release/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $META_ROOT/tools/release/publish-release-catalog.sh"
fi
declare -A PROJECT_NAMES
declare -A OLD_VERSIONS
declare -A BRANCHES
declare -A REPO_KINDS
for repo in "${REPOS[@]}"; do
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
if [[ -f "$repo/pyproject.toml" ]]; then
REPO_KINDS["$repo"]="package"
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
else
REPO_KINDS["$repo"]="tag-only"
PROJECT_NAMES["$repo"]="$(basename "$repo")"
OLD_VERSIONS["$repo"]="tag-only"
fi
if [[ -n "$BRANCH" ]]; then
BRANCHES["$repo"]="$BRANCH"
else
BRANCHES["$repo"]="$(git -C "$repo" symbolic-ref --quiet --short HEAD || true)"
fi
[[ -n "${BRANCHES[$repo]}" ]] || fail "cannot infer branch for $repo; pass --branch <name>"
git -C "$repo" check-ref-format "refs/heads/${BRANCHES[$repo]}" || fail "invalid branch name ${BRANCHES[$repo]} for $repo"
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
fi
done
if [[ -z "$TARGET_VERSION" ]]; then
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
for repo in "${PACKAGE_REPOS[@]}"; do
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
fi
done
TARGET_VERSION="$(bump_version "$BASE_VERSION" "$BUMP")"
fi
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
for repo in "${PACKAGE_REPOS[@]}"; do
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
continue
fi
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
fi
done
TAG="v$TARGET_VERSION"
COMMIT_MESSAGE="${COMMIT_MESSAGE:-Release $TAG}"
TAG_MESSAGE="${TAG_MESSAGE:-$COMMIT_MESSAGE}"
git -C "$ROOT" check-ref-format "refs/tags/$TAG" || fail "invalid tag name: $TAG"
for repo in "${REPOS[@]}"; do
if git -C "$repo" rev-parse --quiet --verify "refs/tags/$TAG" >/dev/null; then
fail "local tag already exists in ${PROJECT_NAMES[$repo]}: $TAG"
fi
set +e
timeout "$GIT_REMOTE_TIMEOUT" git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
ls_remote_status=$?
set -e
if [[ "$ls_remote_status" -eq 0 ]]; then
fail "remote tag already exists for ${PROJECT_NAMES[$repo]} on $REMOTE: $TAG"
elif [[ "$ls_remote_status" -ne 2 ]]; then
fail "could not check remote tags for ${PROJECT_NAMES[$repo]} on $REMOTE"
fi
done
echo "Release plan:"
echo " version: $TARGET_VERSION"
echo " tag: $TAG"
echo " remote: $REMOTE"
echo " commit message: $COMMIT_MESSAGE"
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
else
echo " release lock: skipped"
fi
echo " migration audit: $MIGRATION_AUDIT_MODE"
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
else
echo " web catalog: skipped"
fi
echo
for repo in "${REPOS[@]}"; do
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
else
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
fi
git -C "$repo" status --short
echo
done
run_migration_release_audit
confirm_release
for repo in "${PACKAGE_REPOS[@]}"; do
if [[ "$DRY_RUN" -eq 1 ]]; then
echo "Would update version files in $repo to $TARGET_VERSION"
else
update_version_files "$repo" "$TARGET_VERSION"
fi
done
refresh_development_webui_lock
for repo in "${MODULE_REPOS[@]}"; do
run git -C "$repo" add -A
if [[ "$DRY_RUN" -eq 0 ]]; then
if git -C "$repo" diff --cached --quiet; then
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
echo "No staged changes for ${PROJECT_NAMES[$repo]}; version is already $TARGET_VERSION, tagging current HEAD."
else
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
fi
else
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
fi
else
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
fi
else
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
fi
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
done
for repo in "${MODULE_REPOS[@]}"; do
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
done
generate_release_lock
run git -C "$ROOT" add -A
if [[ "$DRY_RUN" -eq 0 ]]; then
if git -C "$ROOT" diff --cached --quiet; then
fail "no staged changes for ${PROJECT_NAMES[$ROOT]} after version bump"
fi
fi
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
CATALOG_ARGS=(
"$META_ROOT/tools/release/publish-release-catalog.sh"
--version "$TARGET_VERSION"
--channel "$CATALOG_CHANNEL"
--expires-days "$CATALOG_EXPIRES_DAYS"
--core-root "$ROOT"
--web-root "$WEB_ROOT"
--public-base-url "$CATALOG_PUBLIC_URL"
--npm "$NPM_BIN"
--commit
--tag
--push
--remote "$REMOTE"
)
if [[ -n "$BRANCH" ]]; then
CATALOG_ARGS+=(--branch "$BRANCH")
fi
if [[ -n "$CATALOG_SEQUENCE" ]]; then
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
fi
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
CATALOG_ARGS+=(--build-web)
fi
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
done
if [[ "$DRY_RUN" -eq 1 ]]; then
CATALOG_ARGS+=(--dry-run)
fi
run "${CATALOG_ARGS[@]}"
fi
if [[ "$DRY_RUN" -eq 1 ]]; then
echo "Dry run complete for $TAG across all GovOPlaN repos."
else
echo "Released $TAG across all GovOPlaN repos."
fi

1165
tools/release/release-doctor.py Normal file
View File

File diff suppressed because it is too large Load Diff

459
tools/release/release-migration-audit.py Normal file
View File

@@ -0,0 +1,459 @@
#!/usr/bin/env python
from __future__ import annotations
import argparse
import ast
from dataclasses import dataclass
from datetime import datetime, timezone
import json
import os
from pathlib import Path
from typing import Any
META_ROOT = Path(__file__).resolve().parents[2]
ROOT = Path(os.environ.get("GOVOPLAN_CORE_ROOT", META_ROOT.parent / "govoplan-core")).resolve()
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
MIGRATION_TRACK_RELEASE = "release"
MIGRATION_TRACK_DEV = "dev"
MIGRATION_TRACKS = (MIGRATION_TRACK_RELEASE, MIGRATION_TRACK_DEV)
@dataclass(frozen=True, slots=True)
class Migration:
owner: str
path: Path
revision: str
down_revisions: tuple[str, ...]
depends_on: tuple[str, ...]
branch_labels: tuple[str, ...]
def main() -> int:
parser = argparse.ArgumentParser(
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
)
parser.add_argument(
"--workspace-root",
type=Path,
default=ROOT.parent,
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
)
parser.add_argument(
"--baseline-file",
type=Path,
default=DEFAULT_BASELINE_FILE,
help="JSON file recording released migration heads.",
)
parser.add_argument(
"--track",
choices=MIGRATION_TRACKS,
default=MIGRATION_TRACK_RELEASE,
help="Migration track to audit. The release track is compared to release baselines; the dev track only validates the detailed graph.",
)
parser.add_argument(
"--strict",
action="store_true",
help="Fail when current heads are not recorded in the latest release baseline.",
)
parser.add_argument(
"--strict-if-baseline",
action="store_true",
help="Run in strict mode only after at least one release baseline is recorded.",
)
parser.add_argument(
"--record-release",
metavar="VERSION",
help="Record the current reviewed migration heads as a release baseline.",
)
parser.add_argument(
"--replace-release",
action="store_true",
help="Replace an existing release entry when used with --record-release.",
)
parser.add_argument(
"--squash-plan",
action="store_true",
help="Print the reviewed/manual squash checklist for the current graph.",
)
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
args = parser.parse_args()
migrations = discover_migrations(args.workspace_root, track=args.track)
baseline = load_baseline_file(args.baseline_file)
report = build_report(
migrations,
baseline=baseline,
baseline_file=args.baseline_file,
workspace_root=args.workspace_root,
track=args.track,
)
if args.record_release:
if args.track != MIGRATION_TRACK_RELEASE:
raise SystemExit("--record-release is only valid for --track release")
if report["graph_errors"]:
print_text_report(report)
return 1
baseline = record_release_baseline(
baseline,
report,
release=args.record_release,
replace=args.replace_release,
)
write_baseline_file(args.baseline_file, baseline)
report = build_report(
migrations,
baseline=baseline,
baseline_file=args.baseline_file,
workspace_root=args.workspace_root,
track=args.track,
)
if args.squash_plan:
print_squash_plan(report)
elif args.json:
print(json.dumps(report, indent=2, sort_keys=True))
else:
print_text_report(report)
has_graph_errors = bool(report["graph_errors"])
has_strict_errors = bool(report["strict_errors"])
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
if has_graph_errors or (strict_required and has_strict_errors):
return 1
return 0
def discover_migrations(workspace_root: Path, *, track: str = MIGRATION_TRACK_RELEASE) -> list[Migration]:
migrations: list[Migration] = []
for versions_dir in _migration_version_dirs(workspace_root, track=track):
owner = owner_for_versions_dir(versions_dir)
for path in sorted(versions_dir.glob("*.py")):
if path.name == "__init__.py":
continue
migration = parse_migration_file(owner, path)
if migration is not None:
migrations.append(migration)
return migrations
def _migration_version_dirs(workspace_root: Path, *, track: str = MIGRATION_TRACK_RELEASE) -> list[Path]:
version_dir_name = "dev_versions" if track == MIGRATION_TRACK_DEV else "versions"
candidates = [ROOT / "alembic" / version_dir_name]
for repo in sorted(workspace_root.glob("govoplan-*")):
candidates.extend(sorted(repo.glob(f"src/govoplan_*/backend/migrations/{version_dir_name}")))
seen: set[Path] = set()
result: list[Path] = []
for candidate in candidates:
resolved = candidate.resolve()
if resolved in seen or not candidate.is_dir():
continue
seen.add(resolved)
result.append(candidate)
return result
def owner_for_versions_dir(versions_dir: Path) -> str:
core_locations = {
(ROOT / "alembic" / "versions").resolve(),
(ROOT / "alembic" / "dev_versions").resolve(),
}
if versions_dir.resolve() in core_locations:
return "govoplan-core"
for parent in versions_dir.parents:
if parent.name.startswith("govoplan-"):
return parent.name
return versions_dir.parent.name
def parse_migration_file(owner: str, path: Path) -> Migration | None:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
values: dict[str, Any] = {}
for statement in tree.body:
if isinstance(statement, ast.Assign):
for target in statement.targets:
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "depends_on", "branch_labels"}:
values[target.id] = ast.literal_eval(statement.value)
elif (
isinstance(statement, ast.AnnAssign)
and isinstance(statement.target, ast.Name)
and statement.target.id in {"revision", "down_revision", "depends_on", "branch_labels"}
and statement.value is not None
):
values[statement.target.id] = ast.literal_eval(statement.value)
revision = values.get("revision")
if not isinstance(revision, str):
return None
return Migration(
owner=owner,
path=path,
revision=revision,
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
depends_on=_normalize_revision_tuple(values.get("depends_on")),
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
)
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
if value is None:
return ()
if isinstance(value, str):
return (value,)
if isinstance(value, (list, tuple, set)):
return tuple(str(item) for item in value if item is not None)
return (str(value),)
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
if value is None:
return ()
if isinstance(value, str):
return (value,)
if isinstance(value, (list, tuple, set)):
return tuple(str(item) for item in value)
return (str(value),)
def load_baseline_file(path: Path) -> dict[str, Any]:
if not path.exists():
return {"version": 1, "releases": [], "_missing": True}
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise SystemExit(f"{path} must contain a JSON object")
releases = payload.get("releases")
if not isinstance(releases, list):
raise SystemExit(f"{path} must contain a releases list")
return payload
def build_report(
migrations: list[Migration],
*,
baseline: dict[str, Any],
baseline_file: Path,
workspace_root: Path,
track: str = MIGRATION_TRACK_RELEASE,
) -> dict[str, Any]:
revisions: dict[str, Migration] = {}
graph_errors: list[str] = []
for migration in migrations:
existing = revisions.get(migration.revision)
if existing is not None:
graph_errors.append(
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
)
revisions[migration.revision] = migration
referenced = {
revision
for migration in migrations
for revision in (*migration.down_revisions, *migration.depends_on)
}
for migration in migrations:
for down_revision in migration.down_revisions:
if down_revision not in revisions:
graph_errors.append(
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
)
for dependency in migration.depends_on:
if dependency not in revisions:
graph_errors.append(
f"{migration.revision} references missing depends_on {dependency} in {migration.path}"
)
current_heads = sorted(revision for revision in revisions if revision not in referenced)
current_head_entries = [
{
"owner": revisions[revision].owner,
"revision": revision,
}
for revision in current_heads
]
owner_rows = []
for owner in sorted({migration.owner for migration in migrations}):
owner_migrations = [migration for migration in migrations if migration.owner == owner]
owner_revisions = {migration.revision for migration in owner_migrations}
owner_referenced = {
revision
for migration in owner_migrations
for revision in (*migration.down_revisions, *migration.depends_on)
if revision in owner_revisions
}
owner_heads = sorted(owner_revisions - owner_referenced)
owner_rows.append(
{
"owner": owner,
"migrations": len(owner_migrations),
"heads": owner_heads,
}
)
releases = baseline.get("releases") or []
latest_release = releases[-1] if releases else None
latest_heads = _latest_release_heads(latest_release)
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
strict_errors: list[str] = []
if track == MIGRATION_TRACK_RELEASE:
if baseline.get("_missing"):
strict_errors.append(f"baseline file is missing: {baseline_file}")
if not releases:
strict_errors.append("no released migration baseline has been recorded yet")
if unrecorded_heads:
strict_errors.append(
"current migration heads are not recorded in the latest release baseline: "
+ ", ".join(unrecorded_heads)
)
return {
"workspace_root": str(workspace_root),
"track": track,
"baseline_file": str(baseline_file),
"baseline_file_missing": bool(baseline.get("_missing")),
"release_count": len(releases),
"latest_release": latest_release,
"owners": owner_rows,
"current_heads": current_heads,
"current_head_entries": current_head_entries,
"graph_errors": graph_errors,
"strict_errors": strict_errors,
}
def _latest_release_heads(latest_release: Any) -> set[str]:
if not isinstance(latest_release, dict):
return set()
heads = latest_release.get("heads") or []
graph_heads = latest_release.get("graph_heads") or []
result: set[str] = set()
for head_list in (heads, graph_heads):
if not isinstance(head_list, list):
continue
for entry in head_list:
if isinstance(entry, str):
result.add(entry)
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
result.add(entry["revision"])
owner_heads = latest_release.get("owner_heads") or []
if isinstance(owner_heads, list):
for entry in owner_heads:
if not isinstance(entry, dict):
continue
revisions = entry.get("revisions") or []
if isinstance(revisions, str):
result.add(revisions)
elif isinstance(revisions, list):
result.update(revision for revision in revisions if isinstance(revision, str))
return result
def record_release_baseline(
baseline: dict[str, Any],
report: dict[str, Any],
*,
release: str,
replace: bool,
) -> dict[str, Any]:
payload = {
key: value
for key, value in baseline.items()
if not key.startswith("_")
}
payload.setdefault("version", 1)
releases = list(payload.get("releases") or [])
existing_index = next(
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
None,
)
if existing_index is not None and not replace:
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
entry = {
"release": release,
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"track": MIGRATION_TRACK_RELEASE,
"squash_policy": "reviewed-manual",
"heads": report["current_head_entries"],
"owner_heads": [
{
"owner": owner["owner"],
"revisions": owner["heads"],
}
for owner in report["owners"]
],
}
if existing_index is None:
releases.append(entry)
else:
releases[existing_index] = entry
payload["releases"] = releases
return payload
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def print_text_report(report: dict[str, Any]) -> None:
print("Migration release audit")
print(f" workspace: {report['workspace_root']}")
print(f" track: {report['track']}")
print(f" baseline file: {report['baseline_file']}")
print(f" recorded releases: {report['release_count']}")
print()
print("Owners:")
for owner in report["owners"]:
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
print()
print("Current heads:")
for entry in report["current_head_entries"]:
print(f" {entry['owner']}: {entry['revision']}")
if not report["current_head_entries"]:
print(" -")
if report["graph_errors"]:
print()
print("Graph errors:")
for error in report["graph_errors"]:
print(f" - {error}")
if report["strict_errors"]:
print()
print("Release baseline warnings:")
for error in report["strict_errors"]:
print(f" - {error}")
def print_squash_plan(report: dict[str, Any]) -> None:
print("Manual migration squash plan")
print()
print("Policy:")
print(" - Do not rewrite revision IDs that have shipped to real installations.")
print(" - Keep detailed development migrations on the dev track.")
print(" - Add reviewed release-track baselines or release-to-release step-up migrations.")
print(" - Review generated release shortcuts like normal schema code.")
print(" - Run PostgreSQL migration smoke checks after any squash.")
print()
print("Current owner heads:")
for owner in report["owners"]:
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
print(f" - {owner['owner']}: {heads}")
print()
print("Current Alembic graph heads:")
for entry in report["current_head_entries"]:
print(f" - {entry['owner']}: {entry['revision']}")
if not report["current_head_entries"]:
print(" -")
print()
print("Release steps:")
print(" 1. Decide which dev-track revisions are unreleased and should be folded into the next release shortcut.")
print(" 2. Add reviewed release-track baseline or step-up migrations without deleting the dev-track chain.")
print(" 3. Run tools/release/release-migration-audit.py, tools/release/release-migration-audit.py --track dev, and PostgreSQL release checks.")
print(" 4. Record the reviewed heads with tools/release/release-migration-audit.py --record-release <x.y.z>.")
print(" 5. Cut the release with tools/release/push-release-tag.sh; audit is strict after the first baseline.")
if __name__ == "__main__":
raise SystemExit(main())

42
tools/repo/bootstrap-repositories.py Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Clone GovOPlaN repositories listed in repositories.json.")
parser.add_argument("--check", action="store_true", help="Only report missing repositories.")
parser.add_argument("--parent", type=Path, help="Override checkout parent directory.")
return parser.parse_args()
def main() -> int:
args = parse_args()
manifest = json.loads((ROOT / "repositories.json").read_text())
parent = args.parent or Path(manifest["default_parent"])
missing: list[dict[str, str]] = []
for entry in manifest["repositories"]:
repo = parent / entry["path"]
if repo.exists():
continue
missing.append(entry)
print(f"missing: {entry['name']} -> {repo}")
if not args.check:
subprocess.run(["git", "clone", entry["remote"], str(repo)], check=True)
if args.check:
return 1 if missing else 0
return 0
if __name__ == "__main__":
sys.exit(main())

50
tools/repo/repo-status.py Normal file
View File

@@ -0,0 +1,50 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def run_git(repo: Path, *args: str) -> str:
result = subprocess.run(
["git", "-C", str(repo), *args],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.returncode != 0:
return f"git error: {result.stderr.strip()}"
return result.stdout.strip()
def main() -> int:
manifest = json.loads((ROOT / "repositories.json").read_text())
parent = Path(manifest["default_parent"])
rows: list[tuple[str, str, str, str]] = []
for entry in manifest["repositories"]:
repo = parent / entry["path"]
if not repo.exists():
rows.append((entry["name"], entry["category"], "missing", ""))
continue
branch = run_git(repo, "symbolic-ref", "--quiet", "--short", "HEAD") or "(detached)"
status = run_git(repo, "status", "--short")
state = "dirty" if status else "clean"
rows.append((entry["name"], entry["category"], branch, state))
name_width = max(len(row[0]) for row in rows)
category_width = max(len(row[1]) for row in rows)
for name, category, branch, state in rows:
print(f"{name:<{name_width}} {category:<{category_width}} {branch:<24} {state}")
return 0
if __name__ == "__main__":
sys.exit(main())

7
tools/repo/repo-status.sh Normal file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
PYTHON="${PYTHON:-python3}"
exec "$PYTHON" "$ROOT/tools/repo/repo-status.py" "$@"

74
tools/repo/update-repository-type-notes.py Normal file
View File

@@ -0,0 +1,74 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
START = "<!-- govoplan-repository-type:start -->"
END = "<!-- govoplan-repository-type:end -->"
BLOCK_RE = re.compile(rf"\n?{re.escape(START)}.*?{re.escape(END)}\n?", re.DOTALL)
LEGACY_RE = re.compile(r"\n?\*\*Repository type:\*\* [^\n]+\.\n?", re.IGNORECASE)
def type_block(category: str, subtype: str | None) -> str:
if subtype:
label = f"{category} ({subtype})"
else:
label = category
return f"{START}\n**Repository type:** {label}.\n{END}\n"
def update_readme(path: Path, repo_name: str, category: str, subtype: str | None) -> bool:
block = type_block(category, subtype)
if path.exists():
original = path.read_text()
else:
original = f"# {repo_name}\n\n"
text = BLOCK_RE.sub("\n", original).strip() + "\n"
text = LEGACY_RE.sub("\n", text).strip() + "\n"
lines = text.splitlines()
insert_at = 0
if lines and lines[0].startswith("# "):
insert_at = 1
while insert_at < len(lines) and lines[insert_at].strip() == "":
insert_at += 1
del lines[1:insert_at]
insert_at = 1
else:
lines.insert(0, f"# {repo_name}")
insert_at = 1
lines[insert_at:insert_at] = ["", *block.rstrip("\n").splitlines(), ""]
updated = "\n".join(lines).rstrip() + "\n"
if updated == original:
return False
path.write_text(updated)
return True
def main() -> int:
manifest = json.loads((ROOT / "repositories.json").read_text())
parent = Path(manifest["default_parent"])
changed = 0
missing = 0
for entry in manifest["repositories"]:
repo = parent / entry["path"]
if not repo.exists():
print(f"missing repository: {repo}", file=sys.stderr)
missing += 1
continue
if update_readme(repo / "README.md", entry["name"], entry["category"], entry.get("subtype")):
print(f"updated: {entry['name']}/README.md")
changed += 1
print(f"repository type notes updated: {changed}, missing repositories: {missing}")
return 1 if missing else 0
if __name__ == "__main__":
sys.exit(main())