Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a00ef54821 | |||
| edb4687826 | |||
| fcfe0b69a3 | |||
| 715bdcbebe | |||
| 060f4da751 | |||
| c32951393a | |||
| 7b85d6deae | |||
| 2d2d9e7bc7 | |||
| dc1a250797 | |||
| 7b7cc8ada7 | |||
| 722c9e5d1c | |||
| 63e54a67be | |||
| 12b623bec9 | |||
| b788afcae1 | |||
| 2b0cdf13f3 | |||
| 013e0b883d | |||
| 50f60e4d43 | |||
| ff4b514475 | |||
| ca1206e8c9 | |||
| 94236a7d7e | |||
| 8dd5123aab | |||
| c61abe154c | |||
| 83784ea19d | |||
| 81f0f649b5 | |||
| 0a130962d3 | |||
| 79af252e88 | |||
| 635d25c74c | |||
| 150b720f12 | |||
| a2053518d1 | |||
| 6d391d13bd |
32
.env.example
Normal file
32
.env.example
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
# GovOPlaN self-hosted install configuration.
|
||||||
|
# Copy to a deployment-local .env or secret store. Do not commit populated secrets.
|
||||||
|
|
||||||
|
APP_ENV=production
|
||||||
|
GOVOPLAN_INSTALL_PROFILE=self-hosted
|
||||||
|
MASTER_KEY_B64=<generate-with-govoplan-config-env-template-generate-secrets>
|
||||||
|
|
||||||
|
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||||
|
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||||
|
|
||||||
|
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,docs,ops
|
||||||
|
|
||||||
|
CELERY_ENABLED=true
|
||||||
|
REDIS_URL=redis://127.0.0.1:6379/0
|
||||||
|
CELERY_QUEUES=send_email,append_sent,default
|
||||||
|
|
||||||
|
CORS_ORIGINS=https://govoplan.example.org
|
||||||
|
AUTH_COOKIE_SECURE=true
|
||||||
|
AUTH_COOKIE_SAMESITE=lax
|
||||||
|
AUTH_COOKIE_DOMAIN=
|
||||||
|
|
||||||
|
FILE_STORAGE_BACKEND=local
|
||||||
|
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
|
||||||
|
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
|
||||||
|
|
||||||
|
DEV_AUTO_MIGRATE_ENABLED=false
|
||||||
|
DEV_BOOTSTRAP_ENABLED=false
|
||||||
|
DEV_MAILBOX_API_ENABLED=false
|
||||||
|
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable
|
||||||
35
.gitea/ISSUE_TEMPLATE/bug_report.md
Normal file
35
.gitea/ISSUE_TEMPLATE/bug_report.md
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
---
|
||||||
|
name: "Bug"
|
||||||
|
about: "Report a reproducible defect, regression, or incorrect behavior"
|
||||||
|
title: "[Bug] "
|
||||||
|
labels:
|
||||||
|
- type/bug
|
||||||
|
- status/triage
|
||||||
|
- module/core
|
||||||
|
---
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- Repository:
|
||||||
|
- Area/module:
|
||||||
|
- Affected version or commit:
|
||||||
|
|
||||||
|
## Behavior
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
|
||||||
|
Actual:
|
||||||
|
|
||||||
|
## Reproduction
|
||||||
|
|
||||||
|
1.
|
||||||
|
2.
|
||||||
|
3.
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
Logs, screenshots, traces, or failing test output:
|
||||||
|
|
||||||
|
## Verification Target
|
||||||
|
|
||||||
|
Command or workflow that should pass when fixed:
|
||||||
1
.gitea/ISSUE_TEMPLATE/config.yaml
Normal file
1
.gitea/ISSUE_TEMPLATE/config.yaml
Normal file
@@ -0,0 +1 @@
|
|||||||
|
blank_issues_enabled: false
|
||||||
27
.gitea/ISSUE_TEMPLATE/docs_workflow.md
Normal file
27
.gitea/ISSUE_TEMPLATE/docs_workflow.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
---
|
||||||
|
name: "Docs / workflow"
|
||||||
|
about: "Request documentation, process, or developer workflow changes"
|
||||||
|
title: "[Docs] "
|
||||||
|
labels:
|
||||||
|
- type/docs
|
||||||
|
- status/triage
|
||||||
|
- module/core
|
||||||
|
- area/docs
|
||||||
|
---
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- Repository:
|
||||||
|
- Document or workflow:
|
||||||
|
|
||||||
|
## Current State
|
||||||
|
|
||||||
|
What is missing, unclear, duplicated, or stale?
|
||||||
|
|
||||||
|
## Desired State
|
||||||
|
|
||||||
|
What should the docs or workflow make clear?
|
||||||
|
|
||||||
|
## Verification Target
|
||||||
|
|
||||||
|
How should this be checked?
|
||||||
32
.gitea/ISSUE_TEMPLATE/feature_request.md
Normal file
32
.gitea/ISSUE_TEMPLATE/feature_request.md
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
---
|
||||||
|
name: "Feature"
|
||||||
|
about: "Propose new user-visible behavior or platform capability"
|
||||||
|
title: "[Feature] "
|
||||||
|
labels:
|
||||||
|
- type/feature
|
||||||
|
- status/triage
|
||||||
|
- module/core
|
||||||
|
---
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
What user, operator, or developer problem should this solve?
|
||||||
|
|
||||||
|
## Proposed Capability
|
||||||
|
|
||||||
|
What should exist when this is done?
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
- Owning repository:
|
||||||
|
- Related module repositories:
|
||||||
|
- Extension point or integration boundary:
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Verification Target
|
||||||
|
|
||||||
|
Command, scenario, or UI flow that should prove completion:
|
||||||
28
.gitea/ISSUE_TEMPLATE/task.md
Normal file
28
.gitea/ISSUE_TEMPLATE/task.md
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
---
|
||||||
|
name: "Task"
|
||||||
|
about: "Track implementation, maintenance, or migration work"
|
||||||
|
title: "[Task] "
|
||||||
|
labels:
|
||||||
|
- type/task
|
||||||
|
- status/triage
|
||||||
|
- module/core
|
||||||
|
---
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
What needs to be completed?
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- Owning repository:
|
||||||
|
- In-scope:
|
||||||
|
- Out-of-scope:
|
||||||
|
|
||||||
|
## Checklist
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
|
|
||||||
|
## Verification Target
|
||||||
|
|
||||||
|
Command or manual check:
|
||||||
25
.gitea/ISSUE_TEMPLATE/tech_debt.md
Normal file
25
.gitea/ISSUE_TEMPLATE/tech_debt.md
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
---
|
||||||
|
name: "Tech debt"
|
||||||
|
about: "Track cleanup, refactoring, risk reduction, or deferred engineering work"
|
||||||
|
title: "[Debt] "
|
||||||
|
labels:
|
||||||
|
- type/debt
|
||||||
|
- status/triage
|
||||||
|
- module/core
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current Cost
|
||||||
|
|
||||||
|
What does this make harder, riskier, slower, or more fragile?
|
||||||
|
|
||||||
|
## Desired Shape
|
||||||
|
|
||||||
|
What should the code, tests, or architecture look like afterwards?
|
||||||
|
|
||||||
|
## Constraints
|
||||||
|
|
||||||
|
What behavior, compatibility, or module boundary must be preserved?
|
||||||
|
|
||||||
|
## Verification Target
|
||||||
|
|
||||||
|
Focused checks that should pass:
|
||||||
15
.gitea/PULL_REQUEST_TEMPLATE.md
Normal file
15
.gitea/PULL_REQUEST_TEMPLATE.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
## Issue
|
||||||
|
|
||||||
|
Closes #
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
-
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
Follow-up issues:
|
||||||
46
.gitea/workflows/dependency-audit.yml
Normal file
46
.gitea/workflows/dependency-audit.yml
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
name: Dependency Audit
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
schedule:
|
||||||
|
- cron: "23 3 * * 1"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
dependency-audit:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Configure SSH for release dependencies
|
||||||
|
env:
|
||||||
|
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
chmod 700 ~/.ssh
|
||||||
|
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||||
|
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||||
|
chmod 600 ~/.ssh/known_hosts
|
||||||
|
- name: Install backend dev audit dependencies
|
||||||
|
run: |
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/bin/python -m pip install --upgrade pip
|
||||||
|
.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
|
||||||
|
- name: Install WebUI release dependencies
|
||||||
|
working-directory: webui
|
||||||
|
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||||
|
- name: Run dependency audits
|
||||||
|
run: bash scripts/check-dependency-audits.sh
|
||||||
42
.gitea/workflows/module-matrix.yml
Normal file
42
.gitea/workflows/module-matrix.yml
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
name: Module Matrix
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
module-matrix:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Configure SSH for release dependencies
|
||||||
|
env:
|
||||||
|
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
chmod 700 ~/.ssh
|
||||||
|
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||||
|
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||||
|
chmod 600 ~/.ssh/known_hosts
|
||||||
|
- name: Install backend release dependencies
|
||||||
|
run: |
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/bin/python -m pip install --upgrade pip
|
||||||
|
.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
.venv/bin/python -m pip install '.[dev]'
|
||||||
|
- name: Install WebUI release dependencies with test scripts
|
||||||
|
working-directory: webui
|
||||||
|
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||||
|
- name: Run module matrix and contract tests
|
||||||
|
run: bash scripts/check-module-matrix.sh
|
||||||
41
.gitea/workflows/release-integration.yml
Normal file
41
.gitea/workflows/release-integration.yml
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
name: Release Integration
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release-integration:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Configure SSH for release dependencies
|
||||||
|
env:
|
||||||
|
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.ssh
|
||||||
|
chmod 700 ~/.ssh
|
||||||
|
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||||
|
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||||
|
chmod 600 ~/.ssh/known_hosts
|
||||||
|
- name: Install backend release integration dependencies
|
||||||
|
run: |
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/bin/python -m pip install --upgrade pip
|
||||||
|
.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
.venv/bin/python -m pip install '.[dev]'
|
||||||
|
- name: Install WebUI release dependencies
|
||||||
|
working-directory: webui
|
||||||
|
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||||
|
- name: Run release integration checks
|
||||||
|
run: bash scripts/check-release-integration.sh
|
||||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -136,6 +136,18 @@ dist
|
|||||||
.yarn/install-state.gz
|
.yarn/install-state.gz
|
||||||
.pnp.*
|
.pnp.*
|
||||||
|
|
||||||
|
# Local WebUI test/build scratch directories
|
||||||
|
.component-test-build/
|
||||||
|
.module-test-build/
|
||||||
|
.policy-test-build/
|
||||||
|
.template-preview-test-build/
|
||||||
|
.import-test-build/
|
||||||
|
webui/.component-test-build/
|
||||||
|
webui/.module-test-build/
|
||||||
|
webui/.policy-test-build/
|
||||||
|
webui/.template-preview-test-build/
|
||||||
|
webui/.import-test-build/
|
||||||
|
|
||||||
# ---> Python
|
# ---> Python
|
||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ This repository is the platform runner and shared core for GovOPlaN. It owns the
|
|||||||
|
|
||||||
Sibling module repositories usually used with this repo:
|
Sibling module repositories usually used with this repo:
|
||||||
|
|
||||||
|
- `/mnt/DATA/git/govoplan-access`
|
||||||
- `/mnt/DATA/git/govoplan-files`
|
- `/mnt/DATA/git/govoplan-files`
|
||||||
- `/mnt/DATA/git/govoplan-mail`
|
- `/mnt/DATA/git/govoplan-mail`
|
||||||
- `/mnt/DATA/git/govoplan-campaign`
|
- `/mnt/DATA/git/govoplan-campaign`
|
||||||
@@ -44,5 +45,6 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
- Prefer `rg`, `sed`, and targeted tests over broad recursive scans or full builds.
|
- Prefer `rg`, `sed`, and targeted tests over broad recursive scans or full builds.
|
||||||
- Avoid DataGrid changes unless explicitly requested; it is intentionally brittle and has known deferred work.
|
- Avoid DataGrid changes unless explicitly requested; it is intentionally brittle and has known deferred work.
|
||||||
- Do not add module-to-module imports for optional integrations. Use core registry/capability/module metadata paths.
|
- Do not add module-to-module imports for optional integrations. Use core registry/capability/module metadata paths.
|
||||||
|
- Treat Gitea issues as the canonical backlog and state log. Treat Gitea wiki pages as durable project context mirrored from repository and product docs. Use `docs/GITEA_ISSUES.md` for labels, templates, TODO import, wiki sync, and Codex issue updates.
|
||||||
- Do not keep generated WebUI test folders in git status; they should be ignored and removable.
|
- Do not keep generated WebUI test folders in git status; they should be ignored and removable.
|
||||||
- Do not start persistent dev servers unless the user asks.
|
- Do not start persistent dev servers unless the user asks.
|
||||||
|
|||||||
54
README.md
54
README.md
@@ -1,6 +1,10 @@
|
|||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=module-matrix.yml&actor=0&status=0)
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=release-integration.yml&actor=0&status=0)
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=dependency-audit.yml&actor=0&status=0)
|
||||||
|
|
||||||
# govoplan-core
|
# govoplan-core
|
||||||
|
|
||||||
GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, tenant and RBAC infrastructure, governance policy, audit/auth helpers, module discovery, migration registration, and the shared WebUI shell. Feature code is supplied by installed modules.
|
GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, module discovery, migration orchestration, capability contracts, install/uninstall orchestration, and the shared WebUI shell. Platform and feature behavior is supplied by installed modules.
|
||||||
|
|
||||||
## Repository ownership
|
## Repository ownership
|
||||||
|
|
||||||
@@ -9,22 +13,28 @@ Core owns:
|
|||||||
- `govoplan_core.server.app:app`, the FastAPI entry point used by uvicorn
|
- `govoplan_core.server.app:app`, the FastAPI entry point used by uvicorn
|
||||||
- `GovoplanServerConfig`, module discovery, registry validation, and route aggregation
|
- `GovoplanServerConfig`, module discovery, registry validation, and route aggregation
|
||||||
- SQLAlchemy base/session helpers and module migration registration
|
- SQLAlchemy base/session helpers and module migration registration
|
||||||
- tenant/account/session/RBAC/governance/audit models and services
|
- kernel APIs for platform metadata, module lifecycle, health, and development diagnostics
|
||||||
- core API routes for auth, admin, platform metadata, audit, and system health
|
|
||||||
- `@govoplan/core-webui`, including login, CSRF/API helpers, shell layout, generic UI components, IconRail, DataGrid, access boundaries, and module route/nav contracts
|
- `@govoplan/core-webui`, including login, CSRF/API helpers, shell layout, generic UI components, IconRail, DataGrid, access boundaries, and module route/nav contracts
|
||||||
|
|
||||||
Feature modules own their backend routers, models, migrations, permissions, frontend packages, nav items, and route contributions. Core should not import feature pages directly; it imports module manifests and renders their route contributions.
|
Platform and feature modules own their backend routers, models, migrations,
|
||||||
|
permissions, frontend packages, nav items, and route contributions. Access,
|
||||||
|
tenancy, policy, audit, and admin behavior live in their owning platform
|
||||||
|
modules. Core should not import feature pages directly; it imports module
|
||||||
|
manifests and renders their route contributions.
|
||||||
|
|
||||||
## Governance docs
|
## Governance docs
|
||||||
|
|
||||||
Canonical policy documents live in `docs/`:
|
Canonical policy documents live in `docs/`:
|
||||||
|
|
||||||
- [RBAC_MANIFEST.md](docs/RBAC_MANIFEST.md)
|
- [DOCUMENTATION_MAP.md](docs/DOCUMENTATION_MAP.md)
|
||||||
- [SYSTEM_GOVERNANCE_MANIFEST.md](docs/SYSTEM_GOVERNANCE_MANIFEST.md)
|
- [ACCESS_RBAC_MODEL.md](docs/ACCESS_RBAC_MODEL.md)
|
||||||
|
- [GOVERNANCE_MODEL.md](docs/GOVERNANCE_MODEL.md)
|
||||||
- [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md)
|
- [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md)
|
||||||
|
- [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md)
|
||||||
- [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md)
|
- [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md)
|
||||||
|
|
||||||
Modules may define module-specific permissions and policy behavior, but the platform-level permission model and governance hierarchy belong here.
|
Modules define module-specific permissions and policy behavior. Shared DTOs and
|
||||||
|
composition rules live in core only where they are stable kernel contracts.
|
||||||
|
|
||||||
## Backend development
|
## Backend development
|
||||||
|
|
||||||
@@ -35,7 +45,7 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `access,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation.
|
Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
@@ -55,13 +65,24 @@ ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \
|
|||||||
|
|
||||||
The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`.
|
The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`.
|
||||||
|
|
||||||
The default development SQLite database lives at `runtime/multimailer-dev.db`, alongside other local runtime state.
|
The default development database is PostgreSQL at `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev`. Store the password in `~/.pgpass`. To force the disposable SQLite fallback, run with `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite`; that database lives below `runtime/`.
|
||||||
|
|
||||||
Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running.
|
Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running.
|
||||||
|
|
||||||
If the configured local SQLite database is missing or empty, `govoplan_core.devserver` enables the development bootstrap before loading settings. This creates the schema and the default development login on startup. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
To run the production-like local profile with PostgreSQL, Redis, a Celery
|
||||||
|
worker, explicit module configuration, and persistent local file storage:
|
||||||
|
|
||||||
To verify the effective runtime paths and missing-SQLite bootstrap without starting uvicorn, run the smoke mode:
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
See [dev/production-like/README.md](dev/production-like/README.md) for ports,
|
||||||
|
environment overrides, and cleanup commands.
|
||||||
|
|
||||||
|
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
||||||
|
|
||||||
|
To verify the effective runtime paths and bootstrap behavior without starting uvicorn, run the smoke mode:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
@@ -70,7 +91,16 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
|
|
||||||
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
||||||
|
|
||||||
`requirements-dev.txt` links local `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` checkouts for development. `requirements-release.txt` installs those modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||||
|
|
||||||
|
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
||||||
|
For self-hosted config bootstrap and validation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||||
|
```
|
||||||
|
|
||||||
## WebUI development
|
## WebUI development
|
||||||
|
|
||||||
|
|||||||
@@ -5,13 +5,15 @@ from logging.config import fileConfig
|
|||||||
from alembic import context
|
from alembic import context
|
||||||
from sqlalchemy import engine_from_config, pool
|
from sqlalchemy import engine_from_config, pool
|
||||||
|
|
||||||
from govoplan_core.access.db import models as access_models # noqa: F401 - populate access metadata
|
from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata
|
||||||
|
from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata
|
||||||
|
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||||
from govoplan_core.core.migrations import migration_metadata_plan
|
from govoplan_core.core.migrations import migration_metadata_plan
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_core.db import models # noqa: F401 - populate core metadata
|
|
||||||
from govoplan_core.server.default_config import get_server_config
|
from govoplan_core.server.default_config import get_server_config
|
||||||
from govoplan_core.server.registry import build_platform_registry
|
from govoplan_core.server.registry import build_platform_registry
|
||||||
from govoplan_core.settings import settings
|
from govoplan_core.settings import settings
|
||||||
|
from govoplan_core.tenancy.scope import scope_registry
|
||||||
|
|
||||||
config = context.config
|
config = context.config
|
||||||
database_url = config.attributes.get("database_url") or settings.database_url
|
database_url = config.attributes.get("database_url") or settings.database_url
|
||||||
@@ -23,11 +25,13 @@ if config.config_file_name is not None:
|
|||||||
|
|
||||||
def _target_metadata():
|
def _target_metadata():
|
||||||
server_config = get_server_config()
|
server_config = get_server_config()
|
||||||
|
enabled_modules = config.attributes.get("enabled_modules", server_config.enabled_modules)
|
||||||
|
manifest_factories = config.attributes.get("manifest_factories", server_config.manifest_factories)
|
||||||
registry = build_platform_registry(
|
registry = build_platform_registry(
|
||||||
server_config.enabled_modules,
|
enabled_modules,
|
||||||
manifest_factories=server_config.manifest_factories,
|
manifest_factories=manifest_factories,
|
||||||
)
|
)
|
||||||
plan = migration_metadata_plan(registry, extra_metadata=(Base.metadata,))
|
plan = migration_metadata_plan(registry, extra_metadata=(scope_registry.metadata, Base.metadata))
|
||||||
return tuple(dict.fromkeys(plan.metadata))
|
return tuple(dict.fromkeys(plan.metadata))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -94,10 +94,10 @@ def _scrub_policy_column(table_name: str) -> None:
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
inspector = sa.inspect(op.get_bind())
|
inspector = sa.inspect(op.get_bind())
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("system_settings", "tenants"):
|
for table_name in ("core_system_settings", "tenancy_tenants"):
|
||||||
if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}:
|
if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}:
|
||||||
_scrub_settings_table(table_name)
|
_scrub_settings_table(table_name)
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}:
|
if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}:
|
||||||
_scrub_policy_column(table_name)
|
_scrub_policy_column(table_name)
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ def upgrade() -> None:
|
|||||||
batch_op.add_column(sa.Column("locked_by_user_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("locked_by_user_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
op.f("fk_campaign_versions_locked_by_user_id_users"),
|
op.f("fk_campaign_versions_locked_by_user_id_users"),
|
||||||
"users",
|
"access_users",
|
||||||
["locked_by_user_id"],
|
["locked_by_user_id"],
|
||||||
["id"],
|
["id"],
|
||||||
ondelete="SET NULL",
|
ondelete="SET NULL",
|
||||||
|
|||||||
@@ -17,67 +17,67 @@ depends_on = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.add_column(sa.Column("auth_provider", sa.String(length=50), nullable=False, server_default="local"))
|
batch_op.add_column(sa.Column("auth_provider", sa.String(length=50), nullable=False, server_default="local"))
|
||||||
batch_op.add_column(sa.Column("password_hash", sa.String(length=500), nullable=True))
|
batch_op.add_column(sa.Column("password_hash", sa.String(length=500), nullable=True))
|
||||||
batch_op.add_column(sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
batch_op.add_column(sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"user_group_memberships",
|
"access_user_group_memberships",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("group_id", sa.String(length=36), nullable=False),
|
sa.Column("group_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"),
|
sa.UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_user_group_memberships_tenant_id"), "user_group_memberships", ["tenant_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_tenant_id"), "access_user_group_memberships", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_user_group_memberships_user_id"), "user_group_memberships", ["user_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_user_id"), "access_user_group_memberships", ["user_id"])
|
||||||
op.create_index(op.f("ix_user_group_memberships_group_id"), "user_group_memberships", ["group_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_group_id"), "access_user_group_memberships", ["group_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"user_role_assignments",
|
"access_user_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"),
|
sa.UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_user_role_assignments_tenant_id"), "user_role_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_tenant_id"), "access_user_role_assignments", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_user_role_assignments_user_id"), "user_role_assignments", ["user_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_user_id"), "access_user_role_assignments", ["user_id"])
|
||||||
op.create_index(op.f("ix_user_role_assignments_role_id"), "user_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_role_id"), "access_user_role_assignments", ["role_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"group_role_assignments",
|
"access_group_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("group_id", sa.String(length=36), nullable=False),
|
sa.Column("group_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"),
|
sa.UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_group_role_assignments_tenant_id"), "group_role_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_tenant_id"), "access_group_role_assignments", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_group_role_assignments_group_id"), "group_role_assignments", ["group_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_group_id"), "access_group_role_assignments", ["group_id"])
|
||||||
op.create_index(op.f("ix_group_role_assignments_role_id"), "group_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_role_id"), "access_group_role_assignments", ["role_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"auth_sessions",
|
"access_auth_sessions",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
@@ -89,42 +89,42 @@ def upgrade() -> None:
|
|||||||
sa.Column("ip_address", sa.String(length=100), nullable=True),
|
sa.Column("ip_address", sa.String(length=100), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("token_hash"),
|
sa.UniqueConstraint("token_hash"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_auth_sessions_tenant_id"), "auth_sessions", ["tenant_id"])
|
op.create_index(op.f("ix_access_auth_sessions_tenant_id"), "access_auth_sessions", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_auth_sessions_user_id"), "auth_sessions", ["user_id"])
|
op.create_index(op.f("ix_access_auth_sessions_user_id"), "access_auth_sessions", ["user_id"])
|
||||||
op.create_index(op.f("ix_auth_sessions_token_hash"), "auth_sessions", ["token_hash"])
|
op.create_index(op.f("ix_access_auth_sessions_token_hash"), "access_auth_sessions", ["token_hash"])
|
||||||
op.create_index(op.f("ix_auth_sessions_expires_at"), "auth_sessions", ["expires_at"])
|
op.create_index(op.f("ix_access_auth_sessions_expires_at"), "access_auth_sessions", ["expires_at"])
|
||||||
op.create_index(op.f("ix_auth_sessions_revoked_at"), "auth_sessions", ["revoked_at"])
|
op.create_index(op.f("ix_access_auth_sessions_revoked_at"), "access_auth_sessions", ["revoked_at"])
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index(op.f("ix_auth_sessions_revoked_at"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_revoked_at"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_expires_at"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_expires_at"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_token_hash"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_token_hash"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_user_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_user_id"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_tenant_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_tenant_id"), table_name="access_auth_sessions")
|
||||||
op.drop_table("auth_sessions")
|
op.drop_table("access_auth_sessions")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_group_role_assignments_role_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_role_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_index(op.f("ix_group_role_assignments_group_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_group_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_index(op.f("ix_group_role_assignments_tenant_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_tenant_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_table("group_role_assignments")
|
op.drop_table("access_group_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_user_role_assignments_role_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_role_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_index(op.f("ix_user_role_assignments_user_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_user_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_index(op.f("ix_user_role_assignments_tenant_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_tenant_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_table("user_role_assignments")
|
op.drop_table("access_user_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_user_group_memberships_group_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_group_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_index(op.f("ix_user_group_memberships_user_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_user_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_index(op.f("ix_user_group_memberships_tenant_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_tenant_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_table("user_group_memberships")
|
op.drop_table("access_user_group_memberships")
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.drop_column("last_login_at")
|
batch_op.drop_column("last_login_at")
|
||||||
batch_op.drop_column("password_hash")
|
batch_op.drop_column("password_hash")
|
||||||
batch_op.drop_column("auth_provider")
|
batch_op.drop_column("auth_provider")
|
||||||
|
|||||||
78
alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py
Normal file
78
alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
"""namespace platform-owned tables
|
||||||
|
|
||||||
|
Revision ID: 2e3f4a5b6c7d
|
||||||
|
Revises: 1b2c3d4e5f70
|
||||||
|
Create Date: 2026-07-09 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "2e3f4a5b6c7d"
|
||||||
|
down_revision = "1b2c3d4e5f70"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
_TABLE_RENAMES = (
|
||||||
|
("tenants", "tenancy_tenants"),
|
||||||
|
("accounts", "access_accounts"),
|
||||||
|
("users", "access_users"),
|
||||||
|
("groups", "access_groups"),
|
||||||
|
("roles", "access_roles"),
|
||||||
|
("system_role_assignments", "access_system_role_assignments"),
|
||||||
|
("user_group_memberships", "access_user_group_memberships"),
|
||||||
|
("user_role_assignments", "access_user_role_assignments"),
|
||||||
|
("group_role_assignments", "access_group_role_assignments"),
|
||||||
|
("api_keys", "access_api_keys"),
|
||||||
|
("auth_sessions", "access_auth_sessions"),
|
||||||
|
("system_settings", "core_system_settings"),
|
||||||
|
("governance_templates", "admin_governance_templates"),
|
||||||
|
("governance_template_assignments", "admin_governance_template_assignments"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||||
|
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
|
old_tables_to_drop: set[str] = set()
|
||||||
|
new_tables_to_drop: set[str] = set()
|
||||||
|
for old_name, new_name in renames:
|
||||||
|
if old_name not in tables or new_name not in tables:
|
||||||
|
continue
|
||||||
|
if _row_count(bind, old_name) == 0:
|
||||||
|
old_tables_to_drop.add(old_name)
|
||||||
|
elif _row_count(bind, new_name) == 0:
|
||||||
|
new_tables_to_drop.add(new_name)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Cannot rename non-empty {old_name} over non-empty {new_name}")
|
||||||
|
|
||||||
|
for old_name, _new_name in reversed(renames):
|
||||||
|
if old_name in old_tables_to_drop:
|
||||||
|
op.drop_table(old_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
for _old_name, new_name in reversed(renames):
|
||||||
|
if new_name in new_tables_to_drop:
|
||||||
|
op.drop_table(new_name)
|
||||||
|
tables.remove(new_name)
|
||||||
|
|
||||||
|
for old_name, new_name in renames:
|
||||||
|
if old_name not in tables:
|
||||||
|
continue
|
||||||
|
op.rename_table(old_name, new_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
tables.add(new_name)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
_rename_tables(_TABLE_RENAMES)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
_rename_tables(tuple((new_name, old_name) for old_name, new_name in reversed(_TABLE_RENAMES)))
|
||||||
@@ -32,7 +32,7 @@ def upgrade() -> None:
|
|||||||
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),
|
sa.UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),
|
||||||
)
|
)
|
||||||
@@ -55,10 +55,10 @@ def upgrade() -> None:
|
|||||||
sa.Column("metadata", sa.JSON(), nullable=True),
|
sa.Column("metadata", sa.JSON(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "current_version_id", "display_path", "filename", "created_by_user_id", "deleted_at"]:
|
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "current_version_id", "display_path", "filename", "created_by_user_id", "deleted_at"]:
|
||||||
@@ -80,9 +80,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),
|
sa.UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),
|
||||||
)
|
)
|
||||||
@@ -101,9 +101,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),
|
sa.UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),
|
||||||
)
|
)
|
||||||
@@ -136,7 +136,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["file_blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["file_version_id"], ["file_versions.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_version_id"], ["file_versions.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),
|
sa.UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),
|
||||||
)
|
)
|
||||||
|
|||||||
111
alembic/versions/3f4a5b6c7d8e_core_change_sequence.py
Normal file
111
alembic/versions/3f4a5b6c7d8e_core_change_sequence.py
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
"""add core change sequence
|
||||||
|
|
||||||
|
Revision ID: 3f4a5b6c7d8e
|
||||||
|
Revises: 2e3f4a5b6c7d
|
||||||
|
Create Date: 2026-07-09 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "3f4a5b6c7d8e"
|
||||||
|
down_revision = "2e3f4a5b6c7d"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence" not in inspector.get_table_names():
|
||||||
|
op.create_table(
|
||||||
|
"core_change_sequence",
|
||||||
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("collection", sa.String(length=150), nullable=False),
|
||||||
|
sa.Column("resource_type", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("resource_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("operation", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("actor_type", sa.String(length=30), nullable=True),
|
||||||
|
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("payload", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence")),
|
||||||
|
)
|
||||||
|
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")}
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"module_id",
|
||||||
|
"collection",
|
||||||
|
"resource_type",
|
||||||
|
"resource_id",
|
||||||
|
"operation",
|
||||||
|
"actor_type",
|
||||||
|
"actor_id",
|
||||||
|
"created_at",
|
||||||
|
):
|
||||||
|
name = op.f(f"ix_core_change_sequence_{column}")
|
||||||
|
if name not in indexes:
|
||||||
|
op.create_index(name, "core_change_sequence", [column], unique=False)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_core_change_sequence_tenant_module_id", ["tenant_id", "module_id", "id"]),
|
||||||
|
("ix_core_change_sequence_collection_id", ["collection", "id"]),
|
||||||
|
("ix_core_change_sequence_resource", ["module_id", "resource_type", "resource_id"]),
|
||||||
|
):
|
||||||
|
if name not in indexes:
|
||||||
|
op.create_index(name, "core_change_sequence", columns, unique=False)
|
||||||
|
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence_retention_floor" not in inspector.get_table_names():
|
||||||
|
op.create_table(
|
||||||
|
"core_change_sequence_retention_floor",
|
||||||
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("tenant_key", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("collection", sa.String(length=150), nullable=False),
|
||||||
|
sa.Column("min_valid_sequence", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence_retention_floor")),
|
||||||
|
sa.UniqueConstraint("tenant_key", "module_id", "collection", name="uq_core_change_sequence_retention_scope"),
|
||||||
|
)
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")}
|
||||||
|
if "ix_core_change_sequence_retention_scope" not in indexes:
|
||||||
|
op.create_index(
|
||||||
|
"ix_core_change_sequence_retention_scope",
|
||||||
|
"core_change_sequence_retention_floor",
|
||||||
|
["tenant_key", "module_id", "collection"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence_retention_floor" in inspector.get_table_names():
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")}
|
||||||
|
if "ix_core_change_sequence_retention_scope" in indexes:
|
||||||
|
op.drop_index("ix_core_change_sequence_retention_scope", table_name="core_change_sequence_retention_floor")
|
||||||
|
op.drop_table("core_change_sequence_retention_floor")
|
||||||
|
if "core_change_sequence" not in inspector.get_table_names():
|
||||||
|
return
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")}
|
||||||
|
for name in (
|
||||||
|
"ix_core_change_sequence_resource",
|
||||||
|
"ix_core_change_sequence_collection_id",
|
||||||
|
"ix_core_change_sequence_tenant_module_id",
|
||||||
|
op.f("ix_core_change_sequence_created_at"),
|
||||||
|
op.f("ix_core_change_sequence_actor_id"),
|
||||||
|
op.f("ix_core_change_sequence_actor_type"),
|
||||||
|
op.f("ix_core_change_sequence_operation"),
|
||||||
|
op.f("ix_core_change_sequence_resource_id"),
|
||||||
|
op.f("ix_core_change_sequence_resource_type"),
|
||||||
|
op.f("ix_core_change_sequence_collection"),
|
||||||
|
op.f("ix_core_change_sequence_module_id"),
|
||||||
|
op.f("ix_core_change_sequence_tenant_id"),
|
||||||
|
):
|
||||||
|
if name in indexes:
|
||||||
|
op.drop_index(name, table_name="core_change_sequence")
|
||||||
|
op.drop_table("core_change_sequence")
|
||||||
@@ -31,10 +31,10 @@ def upgrade() -> None:
|
|||||||
sa.Column("metadata", sa.JSON(), nullable=True),
|
sa.Column("metadata", sa.JSON(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "path", "created_by_user_id", "deleted_at"]:
|
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "path", "created_by_user_id", "deleted_at"]:
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""rename tenancy scope table to core scopes
|
||||||
|
|
||||||
|
Revision ID: 4f2a9c8e7b6d
|
||||||
|
Revises: 3f4a5b6c7d8e
|
||||||
|
Create Date: 2026-07-10 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "4f2a9c8e7b6d"
|
||||||
|
down_revision = "3f4a5b6c7d8e"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
LEGACY_SCOPE_TABLE = "tenancy_tenants"
|
||||||
|
CORE_SCOPE_TABLE = "core_scopes"
|
||||||
|
LEGACY_SLUG_INDEX = "ix_tenancy_tenants_slug"
|
||||||
|
CORE_SLUG_INDEX = "ix_core_scopes_slug"
|
||||||
|
|
||||||
|
|
||||||
|
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||||
|
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||||
|
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_tables(bind: sa.Connection) -> set[str]:
|
||||||
|
return set(sa.inspect(bind).get_table_names())
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_table_if_empty(bind: sa.Connection, table_name: str) -> bool:
|
||||||
|
if _row_count(bind, table_name) != 0:
|
||||||
|
return False
|
||||||
|
op.drop_table(table_name)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_slug_index(bind: sa.Connection, table_name: str, index_name: str, old_index_name: str) -> None:
|
||||||
|
indexes = {index["name"] for index in sa.inspect(bind).get_indexes(table_name)}
|
||||||
|
if old_index_name in indexes:
|
||||||
|
op.drop_index(old_index_name, table_name=table_name)
|
||||||
|
indexes.remove(old_index_name)
|
||||||
|
if index_name not in indexes:
|
||||||
|
op.create_index(op.f(index_name), table_name, ["slug"], unique=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_scope_table(old_name: str, new_name: str) -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
tables = _scope_tables(bind)
|
||||||
|
if old_name not in tables:
|
||||||
|
if new_name in tables:
|
||||||
|
_ensure_slug_index(bind, new_name, CORE_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else LEGACY_SLUG_INDEX, LEGACY_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else CORE_SLUG_INDEX)
|
||||||
|
return
|
||||||
|
|
||||||
|
if new_name in tables:
|
||||||
|
if _drop_table_if_empty(bind, new_name):
|
||||||
|
tables.remove(new_name)
|
||||||
|
elif _drop_table_if_empty(bind, old_name):
|
||||||
|
_ensure_slug_index(bind, new_name, CORE_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else LEGACY_SLUG_INDEX, LEGACY_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else CORE_SLUG_INDEX)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Cannot reconcile non-empty {old_name} over non-empty {new_name}")
|
||||||
|
|
||||||
|
if old_name in tables and new_name not in tables:
|
||||||
|
op.rename_table(old_name, new_name)
|
||||||
|
_ensure_slug_index(bind, new_name, CORE_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else LEGACY_SLUG_INDEX, LEGACY_SLUG_INDEX if new_name == CORE_SCOPE_TABLE else CORE_SLUG_INDEX)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
_rename_scope_table(LEGACY_SCOPE_TABLE, CORE_SCOPE_TABLE)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
_rename_scope_table(CORE_SCOPE_TABLE, LEGACY_SCOPE_TABLE)
|
||||||
@@ -24,7 +24,7 @@ def upgrade() -> None:
|
|||||||
batch_op.add_column(sa.Column("user_locked_by_user_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("user_locked_by_user_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_campaign_versions_user_locked_by_user_id_users",
|
"fk_campaign_versions_user_locked_by_user_id_users",
|
||||||
"users",
|
"access_users",
|
||||||
["user_locked_by_user_id"],
|
["user_locked_by_user_id"],
|
||||||
["id"],
|
["id"],
|
||||||
ondelete="SET NULL",
|
ondelete="SET NULL",
|
||||||
|
|||||||
@@ -62,25 +62,25 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
user_columns = {column["name"] for column in inspector.get_columns("users")}
|
user_columns = {column["name"] for column in inspector.get_columns("access_users")}
|
||||||
# Base.metadata.create_all() from a newer application can create brand-new
|
# Base.metadata.create_all() from a newer application can create brand-new
|
||||||
# tables while leaving existing tables unaltered. Repair that known drift by
|
# tables while leaving existing tables unaltered. Repair that known drift by
|
||||||
# removing only empty, unreferenced administration tables before applying the
|
# removing only empty, unreferenced administration tables before applying the
|
||||||
# real migration. A non-empty table is never guessed at or discarded.
|
# real migration. A non-empty table is never guessed at or discarded.
|
||||||
if "account_id" not in user_columns and "system_role_assignments" in tables:
|
if "account_id" not in user_columns and "access_system_role_assignments" in tables:
|
||||||
count = bind.execute(sa.text("SELECT COUNT(*) FROM system_role_assignments")).scalar_one()
|
count = bind.execute(sa.text("SELECT COUNT(*) FROM access_system_role_assignments")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
raise RuntimeError("Cannot reconcile non-empty create_all system_role_assignments table")
|
raise RuntimeError("Cannot reconcile non-empty create_all system_role_assignments table")
|
||||||
op.drop_table("system_role_assignments")
|
op.drop_table("access_system_role_assignments")
|
||||||
tables.remove("system_role_assignments")
|
tables.remove("access_system_role_assignments")
|
||||||
if "account_id" not in user_columns and "accounts" in tables:
|
if "account_id" not in user_columns and "access_accounts" in tables:
|
||||||
count = bind.execute(sa.text("SELECT COUNT(*) FROM accounts")).scalar_one()
|
count = bind.execute(sa.text("SELECT COUNT(*) FROM access_accounts")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
raise RuntimeError("Cannot reconcile non-empty create_all accounts table")
|
raise RuntimeError("Cannot reconcile non-empty create_all accounts table")
|
||||||
op.drop_table("accounts")
|
op.drop_table("access_accounts")
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"accounts",
|
"access_accounts",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("email", sa.String(length=320), nullable=False),
|
sa.Column("email", sa.String(length=320), nullable=False),
|
||||||
sa.Column("normalized_email", sa.String(length=320), nullable=False),
|
sa.Column("normalized_email", sa.String(length=320), nullable=False),
|
||||||
@@ -95,29 +95,29 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("normalized_email", name="uq_accounts_normalized_email"),
|
sa.UniqueConstraint("normalized_email", name="uq_accounts_normalized_email"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_accounts_normalized_email"), "accounts", ["normalized_email"])
|
op.create_index(op.f("ix_access_accounts_normalized_email"), "access_accounts", ["normalized_email"])
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"))
|
batch_op.add_column(sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"))
|
||||||
batch_op.add_column(sa.Column("settings", sa.JSON(), nullable=False, server_default="{}"))
|
batch_op.add_column(sa.Column("settings", sa.JSON(), nullable=False, server_default="{}"))
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()))
|
batch_op.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.add_column(sa.Column("is_assignable", sa.Boolean(), nullable=False, server_default=sa.true()))
|
batch_op.add_column(sa.Column("is_assignable", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
||||||
|
|
||||||
users = bind.execute(sa.text(
|
users = bind.execute(sa.text(
|
||||||
"SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM users ORDER BY created_at, id"
|
"SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM access_users ORDER BY created_at, id"
|
||||||
)).mappings().all()
|
)).mappings().all()
|
||||||
|
|
||||||
accounts_by_email: dict[str, str] = {}
|
accounts_by_email: dict[str, str] = {}
|
||||||
@@ -155,7 +155,7 @@ def upgrade() -> None:
|
|||||||
first_system_owner_account_id = account_id
|
first_system_owner_account_id = account_id
|
||||||
|
|
||||||
accounts_table = sa.table(
|
accounts_table = sa.table(
|
||||||
"accounts",
|
"access_accounts",
|
||||||
sa.column("id", sa.String), sa.column("email", sa.String), sa.column("normalized_email", sa.String),
|
sa.column("id", sa.String), sa.column("email", sa.String), sa.column("normalized_email", sa.String),
|
||||||
sa.column("display_name", sa.String), sa.column("is_active", sa.Boolean), sa.column("auth_provider", sa.String),
|
sa.column("display_name", sa.String), sa.column("is_active", sa.Boolean), sa.column("auth_provider", sa.String),
|
||||||
sa.column("password_hash", sa.String), sa.column("password_reset_required", sa.Boolean),
|
sa.column("password_hash", sa.String), sa.column("password_reset_required", sa.Boolean),
|
||||||
@@ -166,54 +166,54 @@ def upgrade() -> None:
|
|||||||
bind.execute(accounts_table.insert(), list(account_rows.values()))
|
bind.execute(accounts_table.insert(), list(account_rows.values()))
|
||||||
for row in users:
|
for row in users:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE users SET account_id = :account_id WHERE id = :user_id"),
|
sa.text("UPDATE access_users SET account_id = :account_id WHERE id = :user_id"),
|
||||||
{"account_id": accounts_by_email[_normalize_email(row["email"])], "user_id": row["id"]},
|
{"account_id": accounts_by_email[_normalize_email(row["email"])], "user_id": row["id"]},
|
||||||
)
|
)
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"UPDATE auth_sessions SET account_id = (SELECT users.account_id FROM users WHERE users.id = auth_sessions.user_id)"
|
"UPDATE access_auth_sessions SET account_id = (SELECT access_users.account_id FROM access_users WHERE access_users.id = access_auth_sessions.user_id)"
|
||||||
))
|
))
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
batch_op.create_foreign_key("fk_users_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
batch_op.create_foreign_key("fk_users_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
||||||
batch_op.create_unique_constraint("uq_users_tenant_account", ["tenant_id", "account_id"])
|
batch_op.create_unique_constraint("uq_users_tenant_account", ["tenant_id", "account_id"])
|
||||||
op.create_index(op.f("ix_users_account_id"), "users", ["account_id"])
|
op.create_index(op.f("ix_access_users_account_id"), "access_users", ["account_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
||||||
op.create_index(op.f("ix_auth_sessions_account_id"), "auth_sessions", ["account_id"])
|
op.create_index(op.f("ix_access_auth_sessions_account_id"), "access_auth_sessions", ["account_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"system_role_assignments",
|
"access_system_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("account_id", sa.String(length=36), nullable=False),
|
sa.Column("account_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["account_id"], ["accounts.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["account_id"], ["access_accounts.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"),
|
sa.UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_system_role_assignments_account_id"), "system_role_assignments", ["account_id"])
|
op.create_index(op.f("ix_access_system_role_assignments_account_id"), "access_system_role_assignments", ["account_id"])
|
||||||
op.create_index(op.f("ix_system_role_assignments_role_id"), "system_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_system_role_assignments_role_id"), "access_system_role_assignments", ["role_id"])
|
||||||
|
|
||||||
roles_table = sa.table(
|
roles_table = sa.table(
|
||||||
"roles",
|
"access_roles",
|
||||||
sa.column("id", sa.String), sa.column("tenant_id", sa.String), sa.column("slug", sa.String),
|
sa.column("id", sa.String), sa.column("tenant_id", sa.String), sa.column("slug", sa.String),
|
||||||
sa.column("name", sa.String), sa.column("description", sa.Text), sa.column("permissions", sa.JSON),
|
sa.column("name", sa.String), sa.column("description", sa.Text), sa.column("permissions", sa.JSON),
|
||||||
sa.column("is_builtin", sa.Boolean), sa.column("is_assignable", sa.Boolean),
|
sa.column("is_builtin", sa.Boolean), sa.column("is_assignable", sa.Boolean),
|
||||||
sa.column("created_at", sa.DateTime(timezone=True)), sa.column("updated_at", sa.DateTime(timezone=True)),
|
sa.column("created_at", sa.DateTime(timezone=True)), sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
)
|
)
|
||||||
now = _now()
|
now = _now()
|
||||||
tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenants")).all()]
|
tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenancy_tenants")).all()]
|
||||||
definitions = _role_definitions()
|
definitions = _role_definitions()
|
||||||
tenant_role_ids: dict[tuple[str, str], str] = {}
|
tenant_role_ids: dict[tuple[str, str], str] = {}
|
||||||
for tenant_id in tenant_ids:
|
for tenant_id in tenant_ids:
|
||||||
for slug, definition in definitions.items():
|
for slug, definition in definitions.items():
|
||||||
existing = bind.execute(
|
existing = bind.execute(
|
||||||
sa.text("SELECT id FROM roles WHERE tenant_id = :tenant_id AND slug = :slug"),
|
sa.text("SELECT id FROM access_roles WHERE tenant_id = :tenant_id AND slug = :slug"),
|
||||||
{"tenant_id": tenant_id, "slug": slug},
|
{"tenant_id": tenant_id, "slug": slug},
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing:
|
if existing:
|
||||||
@@ -253,7 +253,7 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
op.create_index(
|
op.create_index(
|
||||||
"uq_roles_system_slug",
|
"uq_roles_system_slug",
|
||||||
"roles",
|
"access_roles",
|
||||||
["slug"],
|
["slug"],
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=sa.text("tenant_id IS NULL"),
|
sqlite_where=sa.text("tenant_id IS NULL"),
|
||||||
@@ -267,11 +267,11 @@ def upgrade() -> None:
|
|||||||
continue
|
continue
|
||||||
owner_role_id = tenant_role_ids[(row["tenant_id"], "owner")]
|
owner_role_id = tenant_role_ids[(row["tenant_id"], "owner")]
|
||||||
exists = bind.execute(sa.text(
|
exists = bind.execute(sa.text(
|
||||||
"SELECT 1 FROM user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id"
|
"SELECT 1 FROM access_user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id"
|
||||||
), {"tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id}).first()
|
), {"tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id}).first()
|
||||||
if not exists:
|
if not exists:
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"INSERT INTO user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)"
|
"INSERT INTO access_user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)"
|
||||||
), {"id": str(uuid.uuid4()), "tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id, "created_at": now, "updated_at": now})
|
), {"id": str(uuid.uuid4()), "tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id, "created_at": now, "updated_at": now})
|
||||||
|
|
||||||
# Bootstrap rule for existing installations: the earliest active legacy
|
# Bootstrap rule for existing installations: the earliest active legacy
|
||||||
@@ -284,40 +284,40 @@ def upgrade() -> None:
|
|||||||
), None)
|
), None)
|
||||||
if first_system_owner_account_id:
|
if first_system_owner_account_id:
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"INSERT INTO system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)"
|
"INSERT INTO access_system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)"
|
||||||
), {"id": str(uuid.uuid4()), "account_id": first_system_owner_account_id, "role_id": system_owner_role_id, "created_at": now, "updated_at": now})
|
), {"id": str(uuid.uuid4()), "account_id": first_system_owner_account_id, "role_id": system_owner_role_id, "created_at": now, "updated_at": now})
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index("uq_roles_system_slug", table_name="roles")
|
op.drop_index("uq_roles_system_slug", table_name="access_roles")
|
||||||
op.drop_index(op.f("ix_system_role_assignments_role_id"), table_name="system_role_assignments")
|
op.drop_index(op.f("ix_access_system_role_assignments_role_id"), table_name="access_system_role_assignments")
|
||||||
op.drop_index(op.f("ix_system_role_assignments_account_id"), table_name="system_role_assignments")
|
op.drop_index(op.f("ix_access_system_role_assignments_account_id"), table_name="access_system_role_assignments")
|
||||||
op.drop_table("system_role_assignments")
|
op.drop_table("access_system_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_auth_sessions_account_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_account_id"), table_name="access_auth_sessions")
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.drop_constraint("fk_auth_sessions_account_id_accounts", type_="foreignkey")
|
batch_op.drop_constraint("fk_auth_sessions_account_id_accounts", type_="foreignkey")
|
||||||
batch_op.drop_column("account_id")
|
batch_op.drop_column("account_id")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_users_account_id"), table_name="users")
|
op.drop_index(op.f("ix_access_users_account_id"), table_name="access_users")
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.drop_constraint("uq_users_tenant_account", type_="unique")
|
batch_op.drop_constraint("uq_users_tenant_account", type_="unique")
|
||||||
batch_op.drop_constraint("fk_users_account_id_accounts", type_="foreignkey")
|
batch_op.drop_constraint("fk_users_account_id_accounts", type_="foreignkey")
|
||||||
batch_op.drop_column("account_id")
|
batch_op.drop_column("account_id")
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.drop_column("is_assignable")
|
batch_op.drop_column("is_assignable")
|
||||||
batch_op.drop_column("is_builtin")
|
batch_op.drop_column("is_builtin")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.drop_column("is_active")
|
batch_op.drop_column("is_active")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.drop_column("settings")
|
batch_op.drop_column("settings")
|
||||||
batch_op.drop_column("default_locale")
|
batch_op.drop_column("default_locale")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_accounts_normalized_email"), table_name="accounts")
|
op.drop_index(op.f("ix_access_accounts_normalized_email"), table_name="access_accounts")
|
||||||
op.drop_table("accounts")
|
op.drop_table("access_accounts")
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ def upgrade() -> None:
|
|||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
# Reconcile only the empty create_all shape for the newly introduced tables.
|
# Reconcile only the empty create_all shape for the newly introduced tables.
|
||||||
for table_name in ("governance_template_assignments", "governance_templates", "system_settings"):
|
for table_name in ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings"):
|
||||||
if table_name in tables:
|
if table_name in tables:
|
||||||
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
@@ -36,7 +36,7 @@ def upgrade() -> None:
|
|||||||
op.drop_table(table_name)
|
op.drop_table(table_name)
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"system_settings",
|
"core_system_settings",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"),
|
sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"),
|
||||||
sa.Column("allow_tenant_custom_groups", sa.Boolean(), nullable=False, server_default=sa.true()),
|
sa.Column("allow_tenant_custom_groups", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
@@ -50,16 +50,23 @@ def upgrade() -> None:
|
|||||||
now = _now()
|
now = _now()
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"""
|
"""
|
||||||
INSERT INTO system_settings
|
INSERT INTO core_system_settings
|
||||||
(id, default_locale, allow_tenant_custom_groups, allow_tenant_custom_roles,
|
(id, default_locale, allow_tenant_custom_groups, allow_tenant_custom_roles,
|
||||||
allow_tenant_api_keys, settings, created_at, updated_at)
|
allow_tenant_api_keys, settings, created_at, updated_at)
|
||||||
VALUES
|
VALUES
|
||||||
('global', 'en', 1, 1, 1, '{}', :created_at, :updated_at)
|
('global', 'en', :allow_tenant_custom_groups, :allow_tenant_custom_roles,
|
||||||
|
:allow_tenant_api_keys, '{}', :created_at, :updated_at)
|
||||||
"""
|
"""
|
||||||
), {"created_at": now, "updated_at": now})
|
), {
|
||||||
|
"allow_tenant_custom_groups": True,
|
||||||
|
"allow_tenant_custom_roles": True,
|
||||||
|
"allow_tenant_api_keys": True,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
})
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"governance_templates",
|
"admin_governance_templates",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("kind", sa.String(length=20), nullable=False),
|
sa.Column("kind", sa.String(length=20), nullable=False),
|
||||||
sa.Column("slug", sa.String(length=100), nullable=False),
|
sa.Column("slug", sa.String(length=100), nullable=False),
|
||||||
@@ -72,51 +79,51 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"),
|
sa.UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_governance_templates_kind"), "governance_templates", ["kind"])
|
op.create_index(op.f("ix_admin_governance_templates_kind"), "admin_governance_templates", ["kind"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"governance_template_assignments",
|
"admin_governance_template_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("template_id", sa.String(length=36), nullable=False),
|
sa.Column("template_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("mode", sa.String(length=20), nullable=False, server_default="available"),
|
sa.Column("mode", sa.String(length=20), nullable=False, server_default="available"),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["template_id"], ["governance_templates.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["template_id"], ["admin_governance_templates.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"),
|
sa.UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_governance_template_assignments_template_id"), "governance_template_assignments", ["template_id"])
|
op.create_index(op.f("ix_admin_governance_template_assignments_template_id"), "admin_governance_template_assignments", ["template_id"])
|
||||||
op.create_index(op.f("ix_governance_template_assignments_tenant_id"), "governance_template_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_admin_governance_template_assignments_tenant_id"), "admin_governance_template_assignments", ["tenant_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.add_column(sa.Column("allow_custom_groups", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_custom_groups", sa.Boolean(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("allow_custom_roles", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_custom_roles", sa.Boolean(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("allow_api_keys", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_api_keys", sa.Boolean(), nullable=True))
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_groups_system_template_id_governance_templates",
|
"fk_groups_system_template_id_governance_templates",
|
||||||
"governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
"admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_groups_system_template_id"), "groups", ["system_template_id"])
|
op.create_index(op.f("ix_access_groups_system_template_id"), "access_groups", ["system_template_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_roles_system_template_id_governance_templates",
|
"fk_roles_system_template_id_governance_templates",
|
||||||
"governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
"admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_roles_system_template_id"), "roles", ["system_template_id"])
|
op.create_index(op.f("ix_access_roles_system_template_id"), "access_roles", ["system_template_id"])
|
||||||
|
|
||||||
# Existing system owners use system:* and need no data change. Extend the
|
# Existing system owners use system:* and need no data change. Extend the
|
||||||
# read-only built-in auditor role to the newly introduced read scopes.
|
# read-only built-in auditor role to the newly introduced read scopes.
|
||||||
auditor = bind.execute(sa.text(
|
auditor = bind.execute(sa.text(
|
||||||
"SELECT id, permissions FROM roles WHERE tenant_id IS NULL AND slug = 'system_auditor'"
|
"SELECT id, permissions FROM access_roles WHERE tenant_id IS NULL AND slug = 'system_auditor'"
|
||||||
)).mappings().first()
|
)).mappings().first()
|
||||||
if auditor:
|
if auditor:
|
||||||
raw_permissions = auditor["permissions"] or []
|
raw_permissions = auditor["permissions"] or []
|
||||||
@@ -125,7 +132,7 @@ def upgrade() -> None:
|
|||||||
if scope not in permissions:
|
if scope not in permissions:
|
||||||
permissions.append(scope)
|
permissions.append(scope)
|
||||||
roles_table = sa.table(
|
roles_table = sa.table(
|
||||||
"roles",
|
"access_roles",
|
||||||
sa.column("id", sa.String),
|
sa.column("id", sa.String),
|
||||||
sa.column("permissions", sa.JSON),
|
sa.column("permissions", sa.JSON),
|
||||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
@@ -138,26 +145,26 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index(op.f("ix_roles_system_template_id"), table_name="roles")
|
op.drop_index(op.f("ix_access_roles_system_template_id"), table_name="access_roles")
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.drop_constraint("fk_roles_system_template_id_governance_templates", type_="foreignkey")
|
batch_op.drop_constraint("fk_roles_system_template_id_governance_templates", type_="foreignkey")
|
||||||
batch_op.drop_column("system_required")
|
batch_op.drop_column("system_required")
|
||||||
batch_op.drop_column("system_template_id")
|
batch_op.drop_column("system_template_id")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_groups_system_template_id"), table_name="groups")
|
op.drop_index(op.f("ix_access_groups_system_template_id"), table_name="access_groups")
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.drop_constraint("fk_groups_system_template_id_governance_templates", type_="foreignkey")
|
batch_op.drop_constraint("fk_groups_system_template_id_governance_templates", type_="foreignkey")
|
||||||
batch_op.drop_column("system_required")
|
batch_op.drop_column("system_required")
|
||||||
batch_op.drop_column("system_template_id")
|
batch_op.drop_column("system_template_id")
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.drop_column("allow_api_keys")
|
batch_op.drop_column("allow_api_keys")
|
||||||
batch_op.drop_column("allow_custom_roles")
|
batch_op.drop_column("allow_custom_roles")
|
||||||
batch_op.drop_column("allow_custom_groups")
|
batch_op.drop_column("allow_custom_groups")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_governance_template_assignments_tenant_id"), table_name="governance_template_assignments")
|
op.drop_index(op.f("ix_admin_governance_template_assignments_tenant_id"), table_name="admin_governance_template_assignments")
|
||||||
op.drop_index(op.f("ix_governance_template_assignments_template_id"), table_name="governance_template_assignments")
|
op.drop_index(op.f("ix_admin_governance_template_assignments_template_id"), table_name="admin_governance_template_assignments")
|
||||||
op.drop_table("governance_template_assignments")
|
op.drop_table("admin_governance_template_assignments")
|
||||||
op.drop_index(op.f("ix_governance_templates_kind"), table_name="governance_templates")
|
op.drop_index(op.f("ix_admin_governance_templates_kind"), table_name="admin_governance_templates")
|
||||||
op.drop_table("governance_templates")
|
op.drop_table("admin_governance_templates")
|
||||||
op.drop_table("system_settings")
|
op.drop_table("core_system_settings")
|
||||||
|
|||||||
@@ -148,9 +148,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["campaign_id"], ["campaigns.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["campaign_id"], ["campaigns.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("campaign_id", "target_type", "target_id", name="uq_campaign_share_target"),
|
sa.UniqueConstraint("campaign_id", "target_type", "target_id", name="uq_campaign_share_target"),
|
||||||
)
|
)
|
||||||
@@ -160,32 +160,32 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_campaign_shares_target_id", "campaign_shares", ["target_id"])
|
op.create_index("ix_campaign_shares_target_id", "campaign_shares", ["target_id"])
|
||||||
op.create_index("ix_campaign_shares_created_by_user_id", "campaign_shares", ["created_by_user_id"])
|
op.create_index("ix_campaign_shares_created_by_user_id", "campaign_shares", ["created_by_user_id"])
|
||||||
op.create_index("ix_campaign_shares_revoked_at", "campaign_shares", ["revoked_at"])
|
op.create_index("ix_campaign_shares_revoked_at", "campaign_shares", ["revoked_at"])
|
||||||
if "roles" not in tables:
|
if "access_roles" not in tables:
|
||||||
return
|
return
|
||||||
|
|
||||||
rows = bind.execute(sa.text("SELECT id, permissions FROM roles")).mappings().all()
|
rows = bind.execute(sa.text("SELECT id, permissions FROM access_roles")).mappings().all()
|
||||||
for row in rows:
|
for row in rows:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE roles SET permissions = :permissions WHERE id = :id"),
|
sa.text("UPDATE access_roles SET permissions = :permissions WHERE id = :id"),
|
||||||
{"id": row["id"], "permissions": _json(_expand_legacy(_decode(row["permissions"])))},
|
{"id": row["id"], "permissions": _json(_expand_legacy(_decode(row["permissions"])))},
|
||||||
)
|
)
|
||||||
|
|
||||||
for slug, permissions in TENANT_ROLE_PERMISSIONS.items():
|
for slug, permissions in TENANT_ROLE_PERMISSIONS.items():
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"),
|
sa.text("UPDATE access_roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"),
|
||||||
{"slug": slug, "permissions": _json(permissions)},
|
{"slug": slug, "permissions": _json(permissions)},
|
||||||
)
|
)
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
for slug, permissions in SYSTEM_ROLE_PERMISSIONS.items():
|
for slug, permissions in SYSTEM_ROLE_PERMISSIONS.items():
|
||||||
existing = bind.execute(
|
existing = bind.execute(
|
||||||
sa.text("SELECT id FROM roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug}
|
sa.text("SELECT id FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug}
|
||||||
).first()
|
).first()
|
||||||
is_protected = slug == "system_owner"
|
is_protected = slug == "system_owner"
|
||||||
if existing:
|
if existing:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"UPDATE roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE "
|
"UPDATE access_roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE "
|
||||||
"WHERE tenant_id IS NULL AND slug = :slug"
|
"WHERE tenant_id IS NULL AND slug = :slug"
|
||||||
),
|
),
|
||||||
{"slug": slug, "permissions": _json(permissions), "is_builtin": is_protected},
|
{"slug": slug, "permissions": _json(permissions), "is_builtin": is_protected},
|
||||||
@@ -199,7 +199,7 @@ def upgrade() -> None:
|
|||||||
name, description = names[slug]
|
name, description = names[slug]
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"INSERT INTO roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, "
|
"INSERT INTO access_roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, "
|
||||||
"system_template_id, system_required, created_at, updated_at) "
|
"system_template_id, system_required, created_at, updated_at) "
|
||||||
"VALUES (:id, NULL, :slug, :name, :description, :permissions, :is_builtin, TRUE, NULL, FALSE, :created_at, :updated_at)"
|
"VALUES (:id, NULL, :slug, :name, :description, :permissions, :is_builtin, TRUE, NULL, FALSE, :created_at, :updated_at)"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ depends_on = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# ### commands auto generated by Alembic - please adjust! ###
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
op.create_table('tenants',
|
op.create_table('tenancy_tenants',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
@@ -27,7 +27,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tenants'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_tenants'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_tenants_slug'), 'tenants', ['slug'], unique=True)
|
op.create_index(op.f('ix_tenancy_tenants_slug'), 'tenancy_tenants', ['slug'], unique=True)
|
||||||
op.create_table('attachment_blobs',
|
op.create_table('attachment_blobs',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
@@ -38,25 +38,25 @@ def upgrade() -> None:
|
|||||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
||||||
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
||||||
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
||||||
op.create_table('groups',
|
op.create_table('access_groups',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_groups')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_groups')),
|
||||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_groups_tenant_id'), 'groups', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False)
|
||||||
op.create_table('roles',
|
op.create_table('access_roles',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
@@ -64,12 +64,12 @@ def upgrade() -> None:
|
|||||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_roles')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_roles')),
|
||||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_roles_tenant_id'), 'roles', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False)
|
||||||
op.create_table('users',
|
op.create_table('access_users',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('email', sa.String(length=320), nullable=False),
|
sa.Column('email', sa.String(length=320), nullable=False),
|
||||||
@@ -78,13 +78,13 @@ def upgrade() -> None:
|
|||||||
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_users')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_users')),
|
||||||
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=False)
|
op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False)
|
||||||
op.create_index(op.f('ix_users_tenant_id'), 'users', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False)
|
||||||
op.create_table('api_keys',
|
op.create_table('access_api_keys',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
@@ -97,13 +97,13 @@ def upgrade() -> None:
|
|||||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_api_keys'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_api_keys'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_api_keys_prefix'), 'api_keys', ['prefix'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False)
|
||||||
op.create_index(op.f('ix_api_keys_tenant_id'), 'api_keys', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False)
|
||||||
op.create_index(op.f('ix_api_keys_user_id'), 'api_keys', ['user_id'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False)
|
||||||
op.create_table('campaigns',
|
op.create_table('campaigns',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
@@ -115,8 +115,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
||||||
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
||||||
)
|
)
|
||||||
@@ -138,8 +138,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['owner_user_id'], ['users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
||||||
@@ -157,9 +157,9 @@ def upgrade() -> None:
|
|||||||
sa.Column('details', sa.JSON(), nullable=True),
|
sa.Column('details', sa.JSON(), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['api_key_id'], ['api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
||||||
@@ -213,7 +213,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
||||||
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
||||||
)
|
)
|
||||||
@@ -243,7 +243,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
||||||
@@ -327,20 +327,20 @@ def downgrade() -> None:
|
|||||||
op.drop_index(op.f('ix_campaigns_external_id'), table_name='campaigns')
|
op.drop_index(op.f('ix_campaigns_external_id'), table_name='campaigns')
|
||||||
op.drop_index(op.f('ix_campaigns_created_by_user_id'), table_name='campaigns')
|
op.drop_index(op.f('ix_campaigns_created_by_user_id'), table_name='campaigns')
|
||||||
op.drop_table('campaigns')
|
op.drop_table('campaigns')
|
||||||
op.drop_index(op.f('ix_api_keys_user_id'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_user_id'), table_name='access_api_keys')
|
||||||
op.drop_index(op.f('ix_api_keys_tenant_id'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_tenant_id'), table_name='access_api_keys')
|
||||||
op.drop_index(op.f('ix_api_keys_prefix'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_prefix'), table_name='access_api_keys')
|
||||||
op.drop_table('api_keys')
|
op.drop_table('access_api_keys')
|
||||||
op.drop_index(op.f('ix_users_tenant_id'), table_name='users')
|
op.drop_index(op.f('ix_access_users_tenant_id'), table_name='access_users')
|
||||||
op.drop_index(op.f('ix_users_email'), table_name='users')
|
op.drop_index(op.f('ix_access_users_email'), table_name='access_users')
|
||||||
op.drop_table('users')
|
op.drop_table('access_users')
|
||||||
op.drop_index(op.f('ix_roles_tenant_id'), table_name='roles')
|
op.drop_index(op.f('ix_access_roles_tenant_id'), table_name='access_roles')
|
||||||
op.drop_table('roles')
|
op.drop_table('access_roles')
|
||||||
op.drop_index(op.f('ix_groups_tenant_id'), table_name='groups')
|
op.drop_index(op.f('ix_access_groups_tenant_id'), table_name='access_groups')
|
||||||
op.drop_table('groups')
|
op.drop_table('access_groups')
|
||||||
op.drop_index(op.f('ix_attachment_blobs_tenant_id'), table_name='attachment_blobs')
|
op.drop_index(op.f('ix_attachment_blobs_tenant_id'), table_name='attachment_blobs')
|
||||||
op.drop_index(op.f('ix_attachment_blobs_sha256'), table_name='attachment_blobs')
|
op.drop_index(op.f('ix_attachment_blobs_sha256'), table_name='attachment_blobs')
|
||||||
op.drop_table('attachment_blobs')
|
op.drop_table('attachment_blobs')
|
||||||
op.drop_index(op.f('ix_tenants_slug'), table_name='tenants')
|
op.drop_index(op.f('ix_tenancy_tenants_slug'), table_name='tenancy_tenants')
|
||||||
op.drop_table('tenants')
|
op.drop_table('tenancy_tenants')
|
||||||
# ### end Alembic commands ###
|
# ### end Alembic commands ###
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
if "auth_sessions" in tables:
|
if "access_auth_sessions" in tables:
|
||||||
columns = {column["name"] for column in inspector.get_columns("auth_sessions")}
|
columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")}
|
||||||
if "csrf_token_hash" not in columns:
|
if "csrf_token_hash" not in columns:
|
||||||
op.add_column("auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True))
|
op.add_column("access_auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True))
|
||||||
if "mail_server_profiles" not in tables:
|
if "mail_server_profiles" not in tables:
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"mail_server_profiles",
|
"mail_server_profiles",
|
||||||
@@ -40,9 +40,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
|
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["updated_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "slug", name="uq_mail_server_profiles_tenant_slug"),
|
sa.UniqueConstraint("tenant_id", "slug", name="uq_mail_server_profiles_tenant_slug"),
|
||||||
)
|
)
|
||||||
@@ -67,7 +67,7 @@ def downgrade() -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
op.drop_table("mail_server_profiles")
|
op.drop_table("mail_server_profiles")
|
||||||
if "auth_sessions" in inspector.get_table_names():
|
if "access_auth_sessions" in inspector.get_table_names():
|
||||||
columns = {column["name"] for column in inspector.get_columns("auth_sessions")}
|
columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")}
|
||||||
if "csrf_token_hash" in columns:
|
if "csrf_token_hash" in columns:
|
||||||
op.drop_column("auth_sessions", "csrf_token_hash")
|
op.drop_column("access_auth_sessions", "csrf_token_hash")
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ def upgrade() -> None:
|
|||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name not in tables:
|
if table_name not in tables:
|
||||||
continue
|
continue
|
||||||
columns = _columns(inspector, table_name)
|
columns = _columns(inspector, table_name)
|
||||||
@@ -83,6 +83,6 @@ def downgrade() -> None:
|
|||||||
batch.drop_column("scope_type")
|
batch.drop_column("scope_type")
|
||||||
batch.alter_column("tenant_id", existing_type=sa.String(length=36), nullable=False)
|
batch.alter_column("tenant_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
|
|
||||||
for table_name in ("campaigns", "groups", "users"):
|
for table_name in ("campaigns", "access_groups", "access_users"):
|
||||||
if table_name in tables and "mail_profile_policy" in _columns(inspector, table_name):
|
if table_name in tables and "mail_profile_policy" in _columns(inspector, table_name):
|
||||||
op.drop_column(table_name, "mail_profile_policy")
|
op.drop_column(table_name, "mail_profile_policy")
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name not in tables:
|
if table_name not in tables:
|
||||||
continue
|
continue
|
||||||
if "settings" not in _columns(inspector, table_name):
|
if "settings" not in _columns(inspector, table_name):
|
||||||
@@ -36,6 +36,6 @@ def downgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("campaigns", "groups", "users"):
|
for table_name in ("campaigns", "access_groups", "access_users"):
|
||||||
if table_name in tables and "settings" in _columns(inspector, table_name):
|
if table_name in tables and "settings" in _columns(inspector, table_name):
|
||||||
op.drop_column(table_name, "settings")
|
op.drop_column(table_name, "settings")
|
||||||
|
|||||||
5
dev/postgres/.env.example
Normal file
5
dev/postgres/.env.example
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
GOVOPLAN_POSTGRES_DB=govoplan
|
||||||
|
GOVOPLAN_POSTGRES_USER=govoplan
|
||||||
|
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
|
||||||
|
GOVOPLAN_POSTGRES_PORT=55432
|
||||||
|
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan
|
||||||
126
dev/postgres/README.md
Normal file
126
dev/postgres/README.md
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
# PostgreSQL Development Profile
|
||||||
|
|
||||||
|
GovOPlaN development now defaults to PostgreSQL:
|
||||||
|
|
||||||
|
```text
|
||||||
|
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||||
|
```
|
||||||
|
|
||||||
|
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
|
||||||
|
The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if
|
||||||
|
you need a different database.
|
||||||
|
|
||||||
|
## Host PostgreSQL Setup
|
||||||
|
|
||||||
|
Create the default local role and database from a host shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export GOVOPLAN_PG_DB=govoplan_dev
|
||||||
|
export GOVOPLAN_PG_USER=govoplan_dev
|
||||||
|
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
|
||||||
|
echo
|
||||||
|
|
||||||
|
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
|
||||||
|
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
|
||||||
|
else
|
||||||
|
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
|
||||||
|
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
|
||||||
|
|
||||||
|
touch ~/.pgpass
|
||||||
|
chmod 600 ~/.pgpass
|
||||||
|
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
|
||||||
|
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
|
||||||
|
mv ~/.pgpass.tmp ~/.pgpass
|
||||||
|
chmod 600 ~/.pgpass
|
||||||
|
```
|
||||||
|
|
||||||
|
Check access:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
|
||||||
|
-c 'select current_user, current_database();'
|
||||||
|
```
|
||||||
|
|
||||||
|
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
|
||||||
|
available through:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
flatpak-spawn --host /usr/bin/psql --version
|
||||||
|
flatpak-spawn --host /usr/bin/pg_dump --version
|
||||||
|
flatpak-spawn --host /usr/bin/pg_restore --version
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run GovOPlaN Against Host PostgreSQL
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||||
|
--with-dev-data
|
||||||
|
|
||||||
|
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
For the full backend and WebUI launcher:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
To force the old SQLite fallback for a disposable local run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Disposable Docker Testbed
|
||||||
|
|
||||||
|
This testbed is for migration and module-permutation checks. It is not a
|
||||||
|
production deployment profile.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the integration check from the core checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
`--reset-schema` drops and recreates the `public` schema before every module
|
||||||
|
set. Use it only against this disposable database.
|
||||||
|
|
||||||
|
Stop the testbed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
docker compose --env-file .env down
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove all test data:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env down -v
|
||||||
|
```
|
||||||
22
dev/postgres/docker-compose.yml
Normal file
22
dev/postgres/docker-compose.yml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: govoplan-core-postgres-dev
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
|
||||||
|
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
|
||||||
|
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD-SHELL
|
||||||
|
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres-data:
|
||||||
27
dev/production-like/.env.example
Normal file
27
dev/production-like/.env.example
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
APP_ENV=staging
|
||||||
|
GOVOPLAN_INSTALL_PROFILE=production-like
|
||||||
|
MASTER_KEY_B64=
|
||||||
|
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
|
||||||
|
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
|
||||||
|
|
||||||
|
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
REDIS_URL=redis://127.0.0.1:56379/0
|
||||||
|
CELERY_ENABLED=true
|
||||||
|
CELERY_QUEUES=send_email,append_sent,default
|
||||||
|
|
||||||
|
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||||
|
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||||
|
AUTH_COOKIE_SECURE=false
|
||||||
|
FILE_STORAGE_BACKEND=local
|
||||||
|
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||||
|
DEV_AUTO_MIGRATE_ENABLED=false
|
||||||
|
DEV_BOOTSTRAP_ENABLED=true
|
||||||
60
dev/production-like/README.md
Normal file
60
dev/production-like/README.md
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
# Production-Like Development Profile
|
||||||
|
|
||||||
|
This profile runs the shared services that production depends on while keeping
|
||||||
|
API, worker, and WebUI code in the editable local repositories.
|
||||||
|
|
||||||
|
It provides:
|
||||||
|
|
||||||
|
- PostgreSQL with a persistent Docker volume
|
||||||
|
- Redis with append-only persistence
|
||||||
|
- explicit `ENABLED_MODULES`
|
||||||
|
- local durable file storage under `runtime/production-like/files`
|
||||||
|
- a Celery worker process using the same queues as the API
|
||||||
|
|
||||||
|
Start it from the core repository:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The helper wrapper exposes repeatable lifecycle commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/production-like-dev.sh validate-config
|
||||||
|
scripts/production-like-dev.sh seed
|
||||||
|
scripts/production-like-dev.sh start
|
||||||
|
scripts/production-like-dev.sh stop
|
||||||
|
scripts/production-like-dev.sh reset --yes
|
||||||
|
```
|
||||||
|
|
||||||
|
The launcher uses `dev/production-like/.env` when present, otherwise
|
||||||
|
`dev/production-like/.env.example`. Copy the example when you want local port or
|
||||||
|
password changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp dev/production-like/.env.example dev/production-like/.env
|
||||||
|
```
|
||||||
|
|
||||||
|
The API and worker use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
REDIS_URL=redis://127.0.0.1:56379/0
|
||||||
|
CELERY_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
|
||||||
|
containers keep running by default so the next launch is fast. To stop them too:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
To remove all profile data:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/production-like-dev.sh reset --yes
|
||||||
|
```
|
||||||
37
dev/production-like/docker-compose.yml
Normal file
37
dev/production-like/docker-compose.yml
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: govoplan-production-like-postgres
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
|
||||||
|
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
|
||||||
|
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD-SHELL
|
||||||
|
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
container_name: govoplan-production-like-redis
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- redis-data:/data
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres-data:
|
||||||
|
redis-data:
|
||||||
343
docs/ACCESS_RBAC_MODEL.md
Normal file
343
docs/ACCESS_RBAC_MODEL.md
Normal file
@@ -0,0 +1,343 @@
|
|||||||
|
# GovOPlaN RBAC And Resource-Access Model
|
||||||
|
|
||||||
|
**Updated:** 2026-07-11
|
||||||
|
|
||||||
|
## Authorization Equation
|
||||||
|
|
||||||
|
An operation is permitted only when every applicable layer allows it:
|
||||||
|
|
||||||
|
```text
|
||||||
|
effective role/API-key capability
|
||||||
|
AND resource ownership/share access
|
||||||
|
AND workflow state
|
||||||
|
AND active governance/policy constraints
|
||||||
|
```
|
||||||
|
|
||||||
|
RBAC answers what an actor may do. ACLs answer which resource the actor may do
|
||||||
|
it to. Workflow state and policy decide whether the operation is currently
|
||||||
|
valid.
|
||||||
|
|
||||||
|
## Identity And Scope
|
||||||
|
|
||||||
|
```text
|
||||||
|
Account global login identity
|
||||||
|
+- User membership tenant-local identity
|
||||||
|
+- direct tenant roles
|
||||||
|
+- active group memberships
|
||||||
|
| +- inherited tenant roles
|
||||||
|
+- tenant-local API keys
|
||||||
|
|
||||||
|
Account
|
||||||
|
+- direct system-role assignments
|
||||||
|
```
|
||||||
|
|
||||||
|
A browser session has one active tenant membership. System privileges do not
|
||||||
|
silently grant tenant data access. API keys remain tenant-local and receive the
|
||||||
|
intersection of their configured scopes and their owner's live tenant scopes on
|
||||||
|
every request.
|
||||||
|
|
||||||
|
## Wildcards
|
||||||
|
|
||||||
|
```text
|
||||||
|
tenant:* every canonical tenant permission
|
||||||
|
system:* every canonical system permission
|
||||||
|
* legacy alias interpreted as tenant:* only
|
||||||
|
```
|
||||||
|
|
||||||
|
Tenant wildcards never grant system permissions.
|
||||||
|
|
||||||
|
## Canonical Tenant Permissions
|
||||||
|
|
||||||
|
Campaigns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
campaign:read
|
||||||
|
campaign:create
|
||||||
|
campaign:update
|
||||||
|
campaign:copy
|
||||||
|
campaign:archive
|
||||||
|
campaign:delete
|
||||||
|
campaign:share
|
||||||
|
campaign:validate
|
||||||
|
campaign:build
|
||||||
|
campaign:review
|
||||||
|
campaign:send_test
|
||||||
|
campaign:queue
|
||||||
|
campaign:control
|
||||||
|
campaign:send
|
||||||
|
campaign:retry
|
||||||
|
campaign:reconcile
|
||||||
|
```
|
||||||
|
|
||||||
|
Recipients:
|
||||||
|
|
||||||
|
```text
|
||||||
|
recipients:read
|
||||||
|
recipients:write
|
||||||
|
recipients:import
|
||||||
|
recipients:export
|
||||||
|
```
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
```text
|
||||||
|
files:read
|
||||||
|
files:download
|
||||||
|
files:upload
|
||||||
|
files:organize
|
||||||
|
files:share
|
||||||
|
files:delete
|
||||||
|
files:admin
|
||||||
|
```
|
||||||
|
|
||||||
|
Reports and audit:
|
||||||
|
|
||||||
|
```text
|
||||||
|
reports:read
|
||||||
|
reports:export
|
||||||
|
reports:send
|
||||||
|
audit:read
|
||||||
|
```
|
||||||
|
|
||||||
|
Mail servers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
mail_servers:read
|
||||||
|
mail_servers:use
|
||||||
|
mail_servers:test
|
||||||
|
mail_servers:write
|
||||||
|
mail_servers:manage_credentials
|
||||||
|
```
|
||||||
|
|
||||||
|
Tenant administration:
|
||||||
|
|
||||||
|
```text
|
||||||
|
admin:users:read
|
||||||
|
admin:users:create
|
||||||
|
admin:users:update
|
||||||
|
admin:users:suspend
|
||||||
|
|
||||||
|
admin:groups:read
|
||||||
|
admin:groups:write
|
||||||
|
admin:groups:manage_members
|
||||||
|
|
||||||
|
admin:roles:read
|
||||||
|
admin:roles:write
|
||||||
|
admin:roles:assign
|
||||||
|
|
||||||
|
admin:api_keys:read
|
||||||
|
admin:api_keys:create
|
||||||
|
admin:api_keys:revoke
|
||||||
|
|
||||||
|
admin:settings:read
|
||||||
|
admin:settings:write
|
||||||
|
admin:policies:read
|
||||||
|
admin:policies:write
|
||||||
|
```
|
||||||
|
|
||||||
|
## Canonical System Permissions
|
||||||
|
|
||||||
|
```text
|
||||||
|
system:tenants:read
|
||||||
|
system:tenants:create
|
||||||
|
system:tenants:update
|
||||||
|
system:tenants:suspend
|
||||||
|
|
||||||
|
system:accounts:read
|
||||||
|
system:accounts:create
|
||||||
|
system:accounts:update
|
||||||
|
system:accounts:suspend
|
||||||
|
|
||||||
|
system:roles:read
|
||||||
|
system:roles:write
|
||||||
|
system:roles:assign
|
||||||
|
|
||||||
|
system:access:read
|
||||||
|
system:access:assign
|
||||||
|
|
||||||
|
system:audit:read
|
||||||
|
system:settings:read
|
||||||
|
system:settings:write
|
||||||
|
system:governance:read
|
||||||
|
system:governance:write
|
||||||
|
```
|
||||||
|
|
||||||
|
`system:access:*` remains a read/assignment boundary for cross-tenant and
|
||||||
|
system access handling. It is not a separate primary UI area.
|
||||||
|
|
||||||
|
## Default Tenant Roles
|
||||||
|
|
||||||
|
- **Owner:** `tenant:*`. At least one active operational owner must remain.
|
||||||
|
- **Tenant administrator:** settings, policies, users, groups, roles, API keys,
|
||||||
|
and read access to campaigns/files/reports/audit. Real delivery remains
|
||||||
|
separately delegable.
|
||||||
|
- **Administrator:** all tenant permissions for upgraded installations.
|
||||||
|
- **Access administrator:** membership and assignment management within
|
||||||
|
delegation limits.
|
||||||
|
- **Campaign manager:** prepare, validate, and build campaigns; no review
|
||||||
|
approval or real delivery by default.
|
||||||
|
- **Reviewer:** inspect and approve prepared campaign messages.
|
||||||
|
- **Sender:** mock-test, queue, control, send, retry, and reconcile prepared
|
||||||
|
campaigns; can use/test approved mail profiles.
|
||||||
|
- **File manager:** managed file operations without campaign delivery rights.
|
||||||
|
- **Viewer:** read campaigns, recipients, files, and reports.
|
||||||
|
- **Auditor:** read campaigns, recipient evidence, reports, and audit records;
|
||||||
|
export detailed evidence.
|
||||||
|
|
||||||
|
## Default System Roles
|
||||||
|
|
||||||
|
- **System owner:** `system:*`, protected. At least one active account must
|
||||||
|
retain it.
|
||||||
|
- **System administrator:** all specific system permissions, editable and not
|
||||||
|
protected.
|
||||||
|
- **System auditor:** read-only system registry/settings/governance/audit role,
|
||||||
|
editable.
|
||||||
|
|
||||||
|
## Delegation Ceiling
|
||||||
|
|
||||||
|
For role definition, assignment, and API-key creation:
|
||||||
|
|
||||||
|
```text
|
||||||
|
requested scopes subset of actor delegateable scopes
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
1. Tenant roles may contain tenant scopes only.
|
||||||
|
2. System roles may contain system scopes only.
|
||||||
|
3. Definition rights and assignment rights are separate.
|
||||||
|
4. Group definition and group membership management are separate.
|
||||||
|
5. API-key scopes are intersected with the owner's current effective scopes on
|
||||||
|
every request.
|
||||||
|
6. Suspended accounts, users, tenants, or groups stop contributing access
|
||||||
|
immediately.
|
||||||
|
7. Administrative updates are field-sensitive; a user with only status
|
||||||
|
authority cannot change role assignments.
|
||||||
|
|
||||||
|
## Campaign Ownership And ACLs
|
||||||
|
|
||||||
|
A campaign has exactly one owner:
|
||||||
|
|
||||||
|
```text
|
||||||
|
owner user OR owner group
|
||||||
|
```
|
||||||
|
|
||||||
|
Additional active shares may target users or groups with `read` or `write`.
|
||||||
|
|
||||||
|
Resolution:
|
||||||
|
|
||||||
|
- owner user: read and write;
|
||||||
|
- member of owner group: read and write;
|
||||||
|
- explicit read share: read;
|
||||||
|
- explicit write share: read and write;
|
||||||
|
- `tenant:*`: tenant-wide ACL bypass;
|
||||||
|
- ordinary campaign permission without ownership/share: no object access.
|
||||||
|
|
||||||
|
ACLs do not add capabilities. A write share still needs the specific permission
|
||||||
|
for update, validation, review, send, report, retry, or reconciliation.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
The current file access model distinguishes tenant-level file capabilities from
|
||||||
|
space/folder/file ownership:
|
||||||
|
|
||||||
|
| Scope | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `files:read` | browse/read visible file spaces and metadata |
|
||||||
|
| `files:download` | download file content when ACL permits |
|
||||||
|
| `files:upload` | create files in writable spaces |
|
||||||
|
| `files:organize` | create folders, move files, and update metadata where ACL permits |
|
||||||
|
| `files:share` | share files/spaces according to owner and policy rules |
|
||||||
|
| `files:delete` | delete or retire files where ACL permits |
|
||||||
|
| `files:admin` | tenant-wide administration of user/group file spaces |
|
||||||
|
|
||||||
|
External file connections and spaces are additionally constrained by connector
|
||||||
|
policy and owner/group assignment in the files module.
|
||||||
|
|
||||||
|
## Resource Access Explanations
|
||||||
|
|
||||||
|
The access module exposes a diagnostic endpoint for explaining why a principal
|
||||||
|
can see or operate on a concrete resource:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/admin/access/resource-explanation
|
||||||
|
```
|
||||||
|
|
||||||
|
Required query values:
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `user_id` | Tenant membership to explain. The current module UIs pass the signed-in user. |
|
||||||
|
| `resource_type` | Module-owned type such as `file`, `folder`, or `campaign`. |
|
||||||
|
| `resource_id` | Stable module resource identifier. |
|
||||||
|
| `action` | Permission/action being explained, for example `files:file:read`. |
|
||||||
|
| `tenant_id` | Optional tenant override for system/admin contexts. |
|
||||||
|
|
||||||
|
The access module always contributes effective-scope provenance for the action.
|
||||||
|
Installed modules may add resource provenance by exposing a
|
||||||
|
`ResourceAccessExplanationProvider` through a capability consumed by access.
|
||||||
|
Providers should return only facts they own, using these provenance kinds:
|
||||||
|
|
||||||
|
| Kind | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `resource` | The concrete resource or an explicit not-found result. |
|
||||||
|
| `owner` | Matching user/group ownership. |
|
||||||
|
| `share` | Matching explicit user/group/tenant share. |
|
||||||
|
| `policy` | Administrative bypass or policy-derived grant. |
|
||||||
|
| `role` / `right` | Scope and role provenance from access itself. |
|
||||||
|
|
||||||
|
Files currently registers `files.access` and explains file assets plus folders.
|
||||||
|
Persisted folders use their database ID. Folder rows inferred from file paths use
|
||||||
|
a deterministic virtual ID:
|
||||||
|
|
||||||
|
```text
|
||||||
|
virtual-folder:v1:<tenant_id>:<owner_type>:<owner_id>:<base64url(normalized_path)>
|
||||||
|
```
|
||||||
|
|
||||||
|
The files provider validates virtual folder IDs before returning provenance: the
|
||||||
|
tenant and owner must match the resource ID, and at least one active file must
|
||||||
|
exist below the normalized folder path. This keeps virtual folder explanations
|
||||||
|
stable without forcing every inferred tree node to become a stored folder row.
|
||||||
|
|
||||||
|
Campaign currently registers `campaigns.access` and explains the campaign
|
||||||
|
ownership/sharing object itself. Finer-grained campaign sub-objects are tracked
|
||||||
|
separately in `govoplan-campaign#50` until their resource identifiers and access
|
||||||
|
rules are decided.
|
||||||
|
|
||||||
|
Cross-user resource explanation is a policy feature, not a module-local UI
|
||||||
|
detail. Until `govoplan-policy#6` is resolved, module UIs should default to
|
||||||
|
current-user explanation and avoid importing access-admin user-picking
|
||||||
|
components.
|
||||||
|
|
||||||
|
## Mail Servers
|
||||||
|
|
||||||
|
| Scope | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `mail_servers:read` | profile metadata and effective policy visibility |
|
||||||
|
| `mail_servers:use` | use an allowed profile for a campaign or message flow |
|
||||||
|
| `mail_servers:test` | run connectivity tests without revealing secrets |
|
||||||
|
| `mail_servers:write` | create/update profile metadata where policy allows |
|
||||||
|
| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or lower-level credentials where policy allows |
|
||||||
|
|
||||||
|
Reusable encrypted profiles exist. Effective usability is also constrained by
|
||||||
|
hierarchical mail-profile policy, ownership, allowed/forced profile sets,
|
||||||
|
credential inheritance mode, lower-level override switches, and allow/deny
|
||||||
|
patterns.
|
||||||
|
|
||||||
|
## Compatibility Aliases
|
||||||
|
|
||||||
|
Compatibility aliases may exist in backend code for upgraded installations, but
|
||||||
|
new UI and docs should use canonical scopes.
|
||||||
|
|
||||||
|
Current alias direction:
|
||||||
|
|
||||||
|
```text
|
||||||
|
* -> tenant:*
|
||||||
|
system:tenants:write -> create/update/suspend tenant scopes
|
||||||
|
system:access:write -> system access assignment/write scopes
|
||||||
|
```
|
||||||
|
|
||||||
|
A separate `retention:*` family is not currently canonical because retention is
|
||||||
|
managed through system settings and tenant policy scopes. Add it only if
|
||||||
|
retention operation duties need separation from general policy/settings
|
||||||
|
administration.
|
||||||
121
docs/ACTION_EFFECT_AUTOMATION_LAYER.md
Normal file
121
docs/ACTION_EFFECT_AUTOMATION_LAYER.md
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
# Action, Effect, And Automation Layer
|
||||||
|
|
||||||
|
GovOPlaN needs an automation layer because administrative processes will not be
|
||||||
|
only linear screen flows. Workflows, schedules, imports, connectors, policies,
|
||||||
|
and external events all need to request governed actions without bypassing the
|
||||||
|
same safety rules that apply to human users.
|
||||||
|
|
||||||
|
The first implementation should live in `govoplan-workflow` and core contracts.
|
||||||
|
Create a separate `govoplan-automation` module only if action planning,
|
||||||
|
schedulers, rule execution, or cross-module automation become too broad for
|
||||||
|
workflow ownership.
|
||||||
|
|
||||||
|
## Layer Purpose
|
||||||
|
|
||||||
|
The automation layer should provide:
|
||||||
|
|
||||||
|
- a typed action catalogue
|
||||||
|
- a typed effect catalogue
|
||||||
|
- consequence preview before execution
|
||||||
|
- policy and permission checks
|
||||||
|
- idempotent execution
|
||||||
|
- audit and provenance records
|
||||||
|
- retry, quarantine, and manual exception handling
|
||||||
|
- system-actor execution without hiding responsibility
|
||||||
|
|
||||||
|
Automation is not a shortcut around module boundaries. It is a governed caller
|
||||||
|
of module capabilities.
|
||||||
|
|
||||||
|
## Action Definition
|
||||||
|
|
||||||
|
An `ActionDefinition` describes something a human or system actor can request.
|
||||||
|
|
||||||
|
Recommended fields:
|
||||||
|
|
||||||
|
- `action_key`
|
||||||
|
- owning module
|
||||||
|
- input schema
|
||||||
|
- actor requirements and required scopes
|
||||||
|
- required capabilities
|
||||||
|
- policy checks
|
||||||
|
- risk level
|
||||||
|
- reversibility class: reversible, compensatable, corrective-only, or
|
||||||
|
irreversible
|
||||||
|
- expected effects
|
||||||
|
- idempotency key strategy
|
||||||
|
- audit event names
|
||||||
|
- preview provider
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- create a case from a form submission
|
||||||
|
- assign a task
|
||||||
|
- generate a document from a template
|
||||||
|
- send a postbox message
|
||||||
|
- send an email notification
|
||||||
|
- append evidence to records
|
||||||
|
- create a payment request
|
||||||
|
- call an external connector
|
||||||
|
|
||||||
|
## Effect Definition
|
||||||
|
|
||||||
|
An `EffectDefinition` describes the expected and observed result of an action.
|
||||||
|
|
||||||
|
Recommended fields:
|
||||||
|
|
||||||
|
- `effect_key`
|
||||||
|
- affected module and resource references
|
||||||
|
- external system references where applicable
|
||||||
|
- created, changed, deleted, sent, notified, locked, or retained markers
|
||||||
|
- visibility and privacy classification
|
||||||
|
- audit event references
|
||||||
|
- rollback or compensation hints
|
||||||
|
- operator-facing explanation
|
||||||
|
|
||||||
|
Effects should be recorded even when execution fails partially. This makes
|
||||||
|
manual recovery and audit review possible.
|
||||||
|
|
||||||
|
## Execution Model
|
||||||
|
|
||||||
|
The runner should execute an action plan as follows:
|
||||||
|
|
||||||
|
1. Resolve actor context: human, delegated actor, or system actor.
|
||||||
|
2. Validate input schema.
|
||||||
|
3. Resolve required module capabilities.
|
||||||
|
4. Run permission and policy checks.
|
||||||
|
5. Generate a consequence preview.
|
||||||
|
6. Reserve or verify the idempotency key.
|
||||||
|
7. Execute the owning module capability.
|
||||||
|
8. Record observed effects.
|
||||||
|
9. Emit events and audit records.
|
||||||
|
10. Mark the command complete, retryable, quarantined, or requiring manual
|
||||||
|
intervention.
|
||||||
|
|
||||||
|
The runner must never advance workflow state past a required side effect unless
|
||||||
|
the action definition explicitly allows asynchronous completion and the pending
|
||||||
|
state is visible.
|
||||||
|
|
||||||
|
## Failure States
|
||||||
|
|
||||||
|
Automation should use explicit failure states:
|
||||||
|
|
||||||
|
- `blocked`: policy, permission, missing capability, or invalid input prevents
|
||||||
|
execution.
|
||||||
|
- `retryable`: transient transport, timeout, rate-limit, or lock conflict.
|
||||||
|
- `quarantined`: unexpected response, schema mismatch, unsafe partial result,
|
||||||
|
or unknown external state.
|
||||||
|
- `manual_required`: human decision or correction is needed.
|
||||||
|
- `compensation_required`: a later action must correct an already observed
|
||||||
|
side effect.
|
||||||
|
|
||||||
|
These states should be visible in workflow, task, and admin diagnostics.
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Core may own stable DTOs, registry contracts, and generic audit/event hooks.
|
||||||
|
`govoplan-workflow` should own the first runner because workflow is the first
|
||||||
|
module that coordinates cross-module process actions.
|
||||||
|
|
||||||
|
Domain modules own their own action providers. For example, templates own
|
||||||
|
document generation actions, postbox owns postbox message actions, and
|
||||||
|
connectors own external handoff actions.
|
||||||
@@ -24,6 +24,15 @@ network_access = false
|
|||||||
[projects."/mnt/DATA/git/govoplan-core"]
|
[projects."/mnt/DATA/git/govoplan-core"]
|
||||||
trust_level = "trusted"
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
[projects."/mnt/DATA/git/govoplan-access"]
|
||||||
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
[projects."/mnt/DATA/git/govoplan-organizations"]
|
||||||
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
[projects."/mnt/DATA/git/govoplan-identity"]
|
||||||
|
trust_level = "trusted"
|
||||||
|
|
||||||
[projects."/mnt/DATA/git/govoplan-mail"]
|
[projects."/mnt/DATA/git/govoplan-mail"]
|
||||||
trust_level = "trusted"
|
trust_level = "trusted"
|
||||||
|
|
||||||
@@ -42,6 +51,8 @@ Each active repository has an `AGENTS.md` file. These files define ownership, mo
|
|||||||
|
|
||||||
Use `~/.codex/config.toml` for personal defaults, auth/runtime settings, writable roots, and trust decisions. Avoid checking in absolute-path writable roots or model preferences unless they are intentionally team-wide.
|
Use `~/.codex/config.toml` for personal defaults, auth/runtime settings, writable roots, and trust decisions. Avoid checking in absolute-path writable roots or model preferences unless they are intentionally team-wide.
|
||||||
|
|
||||||
|
Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.md` for label setup, TODO import, and Codex issue update commands. Durable docs should describe stable behavior; changing work state belongs on the issue.
|
||||||
|
|
||||||
## Focused Verification
|
## Focused Verification
|
||||||
|
|
||||||
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
||||||
@@ -70,4 +81,5 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi
|
|||||||
- Avoid broad recursive scans and full builds unless the change warrants them.
|
- Avoid broad recursive scans and full builds unless the change warrants them.
|
||||||
- Keep generated build/test folders ignored.
|
- Keep generated build/test folders ignored.
|
||||||
- Keep optional module behavior behind core registry/capability/module metadata boundaries.
|
- Keep optional module behavior behind core registry/capability/module metadata boundaries.
|
||||||
|
- Create or update Gitea issues for TODOs, follow-ups, blockers, and feature requests instead of keeping local backlog files.
|
||||||
- Do not start persistent dev servers unless the user asks.
|
- Do not start persistent dev servers unless the user asks.
|
||||||
|
|||||||
335
docs/CONFIGURATION_PACKAGES.md
Normal file
335
docs/CONFIGURATION_PACKAGES.md
Normal file
@@ -0,0 +1,335 @@
|
|||||||
|
# GovOPlaN Configuration Packages
|
||||||
|
|
||||||
|
Configuration packages are reusable, versioned preconfigurations for a working
|
||||||
|
GovOPlaN capability. They sit above modules: a module installs code,
|
||||||
|
migrations, routes, permissions, and capabilities; a configuration package wires
|
||||||
|
installed modules into a concrete operational setup.
|
||||||
|
|
||||||
|
Example: an application-handling package could configure a public portal form,
|
||||||
|
a case workflow, task creation, a mail template, payment processing, access
|
||||||
|
roles, audit evidence, and the interface bindings between those modules.
|
||||||
|
|
||||||
|
The guiding reference scenario is the government-operations permit journey in
|
||||||
|
`docs/GOVOPLAN_MASTER_ROADMAP.md`: a person applies through the portal,
|
||||||
|
uploads files, receives workflow-driven messages and appointment proposals, has
|
||||||
|
a case opened, gets a permit generated from a template, and completes payment.
|
||||||
|
Configuration packages are the mechanism that should make such processes
|
||||||
|
reusable and safely importable.
|
||||||
|
|
||||||
|
This document is durable architecture context and should be mirrored to the
|
||||||
|
Gitea wiki. Active implementation should be tracked in Gitea issues.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- Import a preconfiguration from a trusted catalog or uploaded package.
|
||||||
|
- Detect which modules, module versions, and capabilities are required.
|
||||||
|
- Detect which configuration fragments must be created, updated, or bound.
|
||||||
|
- Explain import problems and offer actionable resolution paths.
|
||||||
|
- Ask the operator only for data needed to make the package work.
|
||||||
|
- Export an existing working configuration as a reusable package.
|
||||||
|
- Support signed catalogs so approved configurations can be shared, adapted,
|
||||||
|
re-exported, and provided to other installations.
|
||||||
|
|
||||||
|
## Vocabulary
|
||||||
|
|
||||||
|
| Term | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| Module | Installable product code with manifest, migrations, routes, permissions, WebUI, events, and capabilities. |
|
||||||
|
| Configuration | Module-owned settings and resources that make behavior concrete: workflows, forms, templates, roles, policies, connector profiles, routing rules, and defaults. |
|
||||||
|
| Interface | A typed contract between modules: capabilities, commands, events, DTOs, data bindings, and UI extension points. |
|
||||||
|
| Data | Operator- or tenant-provided values needed for a working setup: legal text, sender addresses, account mappings, payment provider credentials, templates, defaults, and optionally reference data. |
|
||||||
|
|
||||||
|
The system should communicate these four layers explicitly:
|
||||||
|
|
||||||
|
```text
|
||||||
|
module = what can exist
|
||||||
|
configuration = what should exist here
|
||||||
|
interface = how configured parts connect
|
||||||
|
data = what the operator must provide for this deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
## Package Model
|
||||||
|
|
||||||
|
A configuration package should be a signed, portable manifest plus module-owned
|
||||||
|
configuration fragments. The package is not a database dump. It should be
|
||||||
|
declarative, idempotent, tenant-aware, and explicit about secrets and external
|
||||||
|
systems.
|
||||||
|
|
||||||
|
Required package metadata:
|
||||||
|
|
||||||
|
- stable package id, name, version, description, publisher, and license
|
||||||
|
- supported GovOPlaN core/module compatibility bounds
|
||||||
|
- required modules with version constraints
|
||||||
|
- optional modules with conditional behavior
|
||||||
|
- required capabilities, commands, event subscriptions, and UI extension points
|
||||||
|
- configuration fragments grouped by owning module
|
||||||
|
- interface bindings between fragments
|
||||||
|
- data requirements to collect from the operator
|
||||||
|
- preflight checks and post-import health checks
|
||||||
|
- migration or transformation rules for older package versions
|
||||||
|
- provenance, export source metadata, and signature metadata
|
||||||
|
|
||||||
|
Configuration fragments are interpreted only by the module that owns them. For
|
||||||
|
example, workflow imports workflow definitions; forms imports form schemas;
|
||||||
|
mail imports mail templates and delivery defaults; payments imports payment
|
||||||
|
profiles; access imports groups, roles, and permission assignments.
|
||||||
|
|
||||||
|
## Module Provider Contract
|
||||||
|
|
||||||
|
Each module that wants to participate should expose a configuration-package
|
||||||
|
provider capability. The core orchestrator should not understand module internals
|
||||||
|
or write module-owned tables directly.
|
||||||
|
|
||||||
|
Baseline provider responsibilities:
|
||||||
|
|
||||||
|
- publish JSON Schema or equivalent typed schemas for importable/exportable
|
||||||
|
fragments
|
||||||
|
- publish data requirements that can be rendered by a generic wizard
|
||||||
|
- validate fragments without applying them
|
||||||
|
- report missing dependencies, missing permissions, conflicts, and unsafe
|
||||||
|
changes
|
||||||
|
- produce a dry-run plan with create, update, bind, skip, and blocked actions
|
||||||
|
- apply fragments idempotently inside module-owned boundaries
|
||||||
|
- export selected module-owned configuration into portable fragments
|
||||||
|
- redact secrets and mark secret placeholders during export
|
||||||
|
- provide post-apply health checks and operator-facing diagnostics
|
||||||
|
|
||||||
|
Provider operations should be versioned. The first stable contract can be small:
|
||||||
|
|
||||||
|
```text
|
||||||
|
describe() -> schemas, supported fragment types, exported scopes
|
||||||
|
preflight(package_fragment, context) -> diagnostics, required_data, plan
|
||||||
|
apply(package_fragment, supplied_data, context) -> result, created_refs
|
||||||
|
export(selection, context) -> fragment, data_placeholders, warnings
|
||||||
|
health(import_result, context) -> diagnostics
|
||||||
|
```
|
||||||
|
|
||||||
|
The initial core contract lives in
|
||||||
|
`govoplan_core.core.configuration_packages`. Modules register providers through
|
||||||
|
module-specific capability keys and implement the `ConfigurationProvider`
|
||||||
|
protocol. The generic `configuration.provider` key names the contract and can be
|
||||||
|
used in package requirements; concrete providers such as `access.configuration`
|
||||||
|
are resolved by the admin package wizard. The first DTO surface includes package
|
||||||
|
manifests, module/capability requirements, module-owned fragments, diagnostics,
|
||||||
|
required operator data, dry-run plan items, apply results, export selections,
|
||||||
|
and export results.
|
||||||
|
|
||||||
|
The initial implementation includes provider-neutral orchestration helpers:
|
||||||
|
|
||||||
|
- `dry_run_configuration_package(...)`
|
||||||
|
- `apply_configuration_package(...)`
|
||||||
|
- `export_configuration_package(...)`
|
||||||
|
|
||||||
|
The first concrete provider is `govoplan_access.backend.configuration_provider`.
|
||||||
|
It supports access-owned `roles`, `groups`, and `group_role_assignments`
|
||||||
|
fragments and applies them idempotently.
|
||||||
|
|
||||||
|
The admin wizard backend starts with these routes:
|
||||||
|
|
||||||
|
- `GET /api/v1/admin/configuration-packages/catalog`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/dry-run`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/apply`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/export`
|
||||||
|
|
||||||
|
## Import Flow
|
||||||
|
|
||||||
|
1. Select a package from a trusted catalog or upload a package file.
|
||||||
|
2. Verify signature, channel, publisher trust, package compatibility, and schema.
|
||||||
|
3. Resolve required modules against installed modules and the module package
|
||||||
|
catalog.
|
||||||
|
4. Produce a dependency plan for modules that must be installed or upgraded.
|
||||||
|
5. Ask the operator for required data using a focused wizard.
|
||||||
|
6. Run dry-run preflight across all participating module providers.
|
||||||
|
7. Show problems grouped by severity, owner module, and resolution.
|
||||||
|
8. Apply module/package changes in a dependency-safe order.
|
||||||
|
9. Run post-import health checks and show the final working status.
|
||||||
|
10. Store import provenance, package version, supplied non-secret metadata, and
|
||||||
|
audit events.
|
||||||
|
|
||||||
|
The wizard should display everything necessary and nothing unnecessary. Generic
|
||||||
|
sections should cover package trust, dependency plan, required data, conflicts,
|
||||||
|
review, and result. Module-specific fields should appear only when the selected
|
||||||
|
package and installed modules require them.
|
||||||
|
|
||||||
|
## Problem Model
|
||||||
|
|
||||||
|
Import diagnostics should be structured and actionable, not free-text logs.
|
||||||
|
Every problem should include severity, owner, affected object, explanation, and
|
||||||
|
at least one suggested resolution when the system can infer it.
|
||||||
|
|
||||||
|
Common diagnostic types:
|
||||||
|
|
||||||
|
- missing module or incompatible module version
|
||||||
|
- missing capability or disabled optional integration
|
||||||
|
- missing operator-supplied data
|
||||||
|
- missing permission or insufficient administrator scope
|
||||||
|
- unresolved interface binding between modules
|
||||||
|
- conflicting existing configuration
|
||||||
|
- external service not reachable or credential test failed
|
||||||
|
- policy or tenant-governance violation
|
||||||
|
- unsafe overwrite, downgrade, or destructive change
|
||||||
|
- schema validation failure
|
||||||
|
- post-import health-check failure
|
||||||
|
|
||||||
|
Resolution actions can include install module, upgrade module, enable
|
||||||
|
integration, provide value, choose existing object, rename imported object,
|
||||||
|
skip optional fragment, replace existing configuration, or cancel import.
|
||||||
|
|
||||||
|
## Export Flow
|
||||||
|
|
||||||
|
Export should be possible from a working tenant or system configuration, but it
|
||||||
|
must be intentional about data boundaries.
|
||||||
|
|
||||||
|
1. Select export scope: system, tenant, module set, workflow, form, case type,
|
||||||
|
portal flow, or another domain object.
|
||||||
|
2. Ask participating modules to export owned fragments and data placeholders.
|
||||||
|
3. Classify exported material as configuration, reference data, sample data,
|
||||||
|
secrets, or deployment-local data.
|
||||||
|
4. Redact secrets by default and replace them with data requirements.
|
||||||
|
5. Let the operator choose whether to include reference/sample data.
|
||||||
|
6. Validate the assembled package by running the same preflight path against a
|
||||||
|
clean target context where possible.
|
||||||
|
7. Sign the package or produce an unsigned development package.
|
||||||
|
8. Optionally publish the package metadata to a configuration catalog.
|
||||||
|
|
||||||
|
Exported packages should record provenance: source GovOPlaN version, module
|
||||||
|
versions, exporter identity, timestamp, selected scope, redactions, and
|
||||||
|
validation status.
|
||||||
|
|
||||||
|
## Catalogs And Trust
|
||||||
|
|
||||||
|
Configuration catalogs should follow the existing module package catalog model:
|
||||||
|
a file-backed or remotely fetched JSON catalog with Ed25519 signatures, channel
|
||||||
|
gating, trusted key ids, and operator-controlled trust policy.
|
||||||
|
|
||||||
|
The initial catalog validator mirrors the module catalog environment model with
|
||||||
|
configuration-specific names:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG=/srv/govoplan/configuration-catalogs/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_URL=https://govoplan.example/configuration-catalogs/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/configuration-catalog-cache/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/configuration-catalog-keyring.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/configuration-catalog-sequences.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Catalog entries should include:
|
||||||
|
|
||||||
|
- package id, version, name, description, publisher, tags, and channel
|
||||||
|
- package artifact URL or repository ref
|
||||||
|
- signature metadata for the package artifact
|
||||||
|
- required modules and version ranges for quick compatibility display
|
||||||
|
- package category such as workflow, portal, case type, governance, connector,
|
||||||
|
report, or tenant baseline
|
||||||
|
- maturity flags such as official, verified, example, local, deprecated
|
||||||
|
|
||||||
|
The catalog verifies that a package is approved to view and import. The package
|
||||||
|
itself must still be signed and validated before use.
|
||||||
|
|
||||||
|
## Example Package
|
||||||
|
|
||||||
|
Application handling through a public portal:
|
||||||
|
|
||||||
|
Required modules:
|
||||||
|
|
||||||
|
- `portal` for the public user-facing entry point
|
||||||
|
- `forms` for the application form and validation
|
||||||
|
- `cases` for the case record and case type
|
||||||
|
- `workflow` for status transitions and automation
|
||||||
|
- `tasks` for internal task creation
|
||||||
|
- `templates` for mail/template rendering
|
||||||
|
- `mail` for delivery profile and outbound message sending
|
||||||
|
- `payments` for provider configuration and payment capture
|
||||||
|
- `access` for roles, groups, and permissions
|
||||||
|
- `audit` for import, case, mail, and payment evidence
|
||||||
|
|
||||||
|
Required configuration:
|
||||||
|
|
||||||
|
- portal route and access policy
|
||||||
|
- form schema, validation rules, confirmation texts, and attachments
|
||||||
|
- case type, fields, lifecycle states, and retention classification
|
||||||
|
- workflow steps, transitions, assignees, and completion triggers
|
||||||
|
- task template and queue/group assignment
|
||||||
|
- mail template and delivery rule
|
||||||
|
- payment product, amount rules, provider profile, and webhook binding
|
||||||
|
- access roles/groups for clerks, reviewers, supervisors, and operators
|
||||||
|
- audit event categories and evidence retention defaults
|
||||||
|
|
||||||
|
Required operator data:
|
||||||
|
|
||||||
|
- tenant or organizational unit
|
||||||
|
- service name and public contact details
|
||||||
|
- portal URL/path and legal imprint/privacy text
|
||||||
|
- mail sender profile and reply-to mailbox
|
||||||
|
- payment provider credentials or test-mode selection
|
||||||
|
- responsible groups or users for task assignment
|
||||||
|
- escalation contacts and deadlines
|
||||||
|
- template wording and localized text overrides
|
||||||
|
|
||||||
|
Potential problems:
|
||||||
|
|
||||||
|
- `payments` is missing or the provider capability is unavailable
|
||||||
|
- mail transport test fails for the selected sender profile
|
||||||
|
- imported role names conflict with existing tenant roles
|
||||||
|
- workflow references a task queue that does not exist
|
||||||
|
- public portal base URL is not configured
|
||||||
|
- required privacy/legal text is empty
|
||||||
|
- webhook endpoint cannot be validated from the payment provider
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
Core should own:
|
||||||
|
|
||||||
|
- package and catalog signature validation
|
||||||
|
- dependency resolution against installed modules and module catalogs
|
||||||
|
- orchestration of provider preflight/apply/export operations
|
||||||
|
- generic import/export APIs and audit envelope
|
||||||
|
- generic wizard shell and problem display components
|
||||||
|
|
||||||
|
Modules should own:
|
||||||
|
|
||||||
|
- schemas and semantics for their own fragments
|
||||||
|
- module-specific validation, apply, export, and health checks
|
||||||
|
- module-specific UI capabilities only when generic generated controls are not
|
||||||
|
sufficient
|
||||||
|
- redaction and classification of module-owned secrets or sensitive data
|
||||||
|
|
||||||
|
Access should own configuration fragments for users, groups, roles,
|
||||||
|
permissions, API keys, and principal mappings. It should not own workflow, mail,
|
||||||
|
payment, form, or portal semantics.
|
||||||
|
|
||||||
|
Admin should expose the operator-facing catalog, import, export, and history
|
||||||
|
screens through admin route contributions. The UI should keep the conceptual
|
||||||
|
layers visible: modules, configuration, interfaces, and data.
|
||||||
|
|
||||||
|
## Implementation Slices
|
||||||
|
|
||||||
|
1. Define package manifest and diagnostic schemas.
|
||||||
|
2. Add core configuration-package provider capability contracts.
|
||||||
|
3. Implement catalog validation and package signature verification.
|
||||||
|
4. Add dry-run orchestration against mock providers.
|
||||||
|
5. Add admin catalog/import wizard screens using provider data requirements.
|
||||||
|
6. Implement export/import providers for access-owned roles/groups first.
|
||||||
|
7. Add providers in workflow/forms/templates/mail/payments/portal/cases/tasks as
|
||||||
|
those modules mature.
|
||||||
|
8. Add round-trip tests: export a known setup, import into a clean tenant,
|
||||||
|
verify health checks.
|
||||||
|
9. Add documentation and field-level help for package authors and operators.
|
||||||
|
|
||||||
|
## Acceptance Criteria For Tracking Issue
|
||||||
|
|
||||||
|
- A Gitea tracking issue exists in `govoplan-core` for the cross-cutting
|
||||||
|
feature.
|
||||||
|
- The issue links module-specific follow-ups when implementation begins.
|
||||||
|
- The first implementation exposes typed contracts rather than direct
|
||||||
|
cross-module imports.
|
||||||
|
- A signed example catalog and unsigned development fixture exist.
|
||||||
|
- A dry-run import can identify required modules, missing data, and conflicts
|
||||||
|
before applying changes.
|
||||||
|
- An export can produce a package with secrets redacted into data requirements.
|
||||||
|
- Admin UI shows a focused wizard and actionable diagnostic list.
|
||||||
|
- Tests cover package validation, signature failure, dependency resolution,
|
||||||
|
provider preflight, export redaction, and import idempotency.
|
||||||
67
docs/DEPENDENCY_AUDITS.md
Normal file
67
docs/DEPENDENCY_AUDITS.md
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Dependency Audits
|
||||||
|
|
||||||
|
GovOPlaN keeps dependency vulnerability checks reproducible but separate from
|
||||||
|
the fast local smoke suite, because both Python and npm audits need network
|
||||||
|
metadata and can fail for newly disclosed advisories without a source change.
|
||||||
|
|
||||||
|
## Local Workflow
|
||||||
|
|
||||||
|
Install the development audit dependency once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Run both backend and WebUI production audits:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The script runs:
|
||||||
|
|
||||||
|
- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||||
|
legacy editable package metadata, deprecated framework constants, and the
|
||||||
|
Starlette `TestClient` deprecation smoke when test dependencies are present
|
||||||
|
- `python -m pip_audit --progress-spinner off`
|
||||||
|
- `npm audit --omit=dev` in `webui`
|
||||||
|
|
||||||
|
For fast local checks without vulnerability metadata lookups, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This is also part of `scripts/check-focused.sh`, so resolver drift and
|
||||||
|
deprecation regressions fail close to the code change that introduced them.
|
||||||
|
|
||||||
|
Override tool paths when testing from a disposable environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHON=/tmp/govoplan-audit/bin/python \
|
||||||
|
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## CI Workflow
|
||||||
|
|
||||||
|
`.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||||
|
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
||||||
|
pull requests, and a weekly schedule.
|
||||||
|
|
||||||
|
The workflow intentionally uses release dependency refs instead of local
|
||||||
|
`file:` or editable sibling paths. Development lockfiles may keep local module
|
||||||
|
links, but release audit results should represent the installable product.
|
||||||
|
|
||||||
|
## Recording Results
|
||||||
|
|
||||||
|
When closing or triaging dependency-audit issues, add a short dated note under
|
||||||
|
`docs/audits/`. Record:
|
||||||
|
|
||||||
|
- the commands that were run
|
||||||
|
- whether Python and npm passed
|
||||||
|
- any advisories accepted as temporary risk
|
||||||
|
- follow-up issue links for required upgrades
|
||||||
424
docs/DEPLOYMENT_OPERATOR_GUIDE.md
Normal file
424
docs/DEPLOYMENT_OPERATOR_GUIDE.md
Normal file
@@ -0,0 +1,424 @@
|
|||||||
|
# GovOPlaN Deployment Operator Guide
|
||||||
|
|
||||||
|
This guide defines the current install/runtime configuration contract and the
|
||||||
|
operator flow for a production-realistic self-hosted deployment. Keep secrets in
|
||||||
|
the deployment environment or a secret manager; do not commit populated `.env`
|
||||||
|
files.
|
||||||
|
|
||||||
|
## Runtime Configuration Contract
|
||||||
|
|
||||||
|
Self-hosted installability follows the staged approach documented in
|
||||||
|
`SELF_HOSTED_INSTALLABILITY.md`: generate an explicit env template, validate it,
|
||||||
|
run production-like rehearsal with Compose-backed dependencies, then use the
|
||||||
|
installer CLI/daemon for package mutation under maintenance mode.
|
||||||
|
|
||||||
|
Generate a deployment-local template:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||||
|
--profile self-hosted \
|
||||||
|
--generate-secrets \
|
||||||
|
--output .env.self-hosted
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate the active shell environment before migration or startup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
set -a
|
||||||
|
. .env.self-hosted
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||||
|
```
|
||||||
|
|
||||||
|
### Required Runtime Identity
|
||||||
|
|
||||||
|
| Setting | Required outside dev | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `APP_ENV` | yes | Runtime profile. Use `prod`, `staging`, or a deployment-specific value outside local development. |
|
||||||
|
| `MASTER_KEY_B64` | yes | Fernet key or base64 encoded 32-byte key used for encrypted module secrets. Rotate through an explicit operator plan. |
|
||||||
|
| `DATABASE_URL` | yes | SQLAlchemy database URL for core and installed modules. SQLite is supported for dev/small installs; PostgreSQL is the preferred production target. |
|
||||||
|
| `ENABLED_MODULES` | yes | Comma-separated startup module set. Keep `tenancy,access` enabled; keep `admin` enabled for operator UI. |
|
||||||
|
|
||||||
|
Generate a local key for a new non-production environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python - <<'PY'
|
||||||
|
from cryptography.fernet import Fernet
|
||||||
|
print(Fernet.generate_key().decode())
|
||||||
|
PY
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database And Migrations
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
||||||
|
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
||||||
|
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||||
|
|
||||||
|
Operator rule: take a database backup before applying migrations or destructive
|
||||||
|
module retirement. For non-SQLite databases, configure deployment-specific
|
||||||
|
backup/restore hooks for the module installer.
|
||||||
|
|
||||||
|
### PostgreSQL Production Target
|
||||||
|
|
||||||
|
PostgreSQL is the primary development and production target. SQLite remains
|
||||||
|
supported only for tiny disposable profiles and unit-test style smoke runs.
|
||||||
|
Production/staging deployments should use a managed PostgreSQL database and
|
||||||
|
explicit migration commands.
|
||||||
|
|
||||||
|
Install the server extra so the `psycopg` driver is available:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Example runtime database URLs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DATABASE_URL='postgresql+psycopg://govoplan:change-me@db.example.internal:5432/govoplan'
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS='postgresql://govoplan:change-me@db.example.internal:5432/govoplan'
|
||||||
|
```
|
||||||
|
|
||||||
|
Use the SQLAlchemy URL for GovOPlaN. Use the pg-tools URL for `pg_dump`,
|
||||||
|
`pg_restore`, and `psql`; these tools do not understand the
|
||||||
|
`postgresql+psycopg://` driver marker.
|
||||||
|
|
||||||
|
Bootstrap or upgrade the schema explicitly during deployment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export APP_ENV=prod
|
||||||
|
export ENABLED_MODULES=tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url "$DATABASE_URL"
|
||||||
|
```
|
||||||
|
|
||||||
|
Backup and restore-check before migration-bearing package changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pg_dump --format=custom \
|
||||||
|
--file "$PWD/runtime/govoplan-$(date +%Y%m%d%H%M%S).dump" \
|
||||||
|
"$GOVOPLAN_DATABASE_URL_PGTOOLS"
|
||||||
|
pg_restore --list "$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump" >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Restore a checked backup to the target database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pg_restore --clean --if-exists \
|
||||||
|
--dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" \
|
||||||
|
"$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump"
|
||||||
|
```
|
||||||
|
|
||||||
|
For local development, create the host database described in
|
||||||
|
`dev/postgres/README.md`, then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||||
|
--with-dev-data
|
||||||
|
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
|
scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
For disposable local validation against a throwaway PostgreSQL instance, use
|
||||||
|
the bundled PostgreSQL testbed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
The integration check runs migrations and startup smoke checks across the
|
||||||
|
standard module permutations. `--reset-schema` is destructive and belongs only
|
||||||
|
on throwaway databases.
|
||||||
|
|
||||||
|
### Broker And Workers
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `REDIS_URL` | `redis://redis:6379/0` | Celery broker/result backend when async workers are enabled. |
|
||||||
|
| `CELERY_ENABLED` | `false` | Local/dev can send synchronously. Production campaign delivery should run workers and set this to `true`. |
|
||||||
|
| `CELERY_QUEUES` | `send_email,append_sent,default` | Queue list expected by worker/process manager definitions. |
|
||||||
|
|
||||||
|
Worker command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m celery -A govoplan_core.celery_app:celery worker \
|
||||||
|
--queues send_email,append_sent,default \
|
||||||
|
--loglevel INFO
|
||||||
|
```
|
||||||
|
|
||||||
|
### Storage
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `FILE_STORAGE_BACKEND` | `local` | Use `local` for dev/small deployments; use object storage when files must scale independently. |
|
||||||
|
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Must live on durable storage and be backed up when `FILE_STORAGE_BACKEND=local`. |
|
||||||
|
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Read-only fallback roots for migrated local files. |
|
||||||
|
| `FILE_STORAGE_S3_ENDPOINT_URL` | empty | Object-store endpoint for the files module. |
|
||||||
|
| `FILE_STORAGE_S3_REGION` | empty | Object-store region. |
|
||||||
|
| `FILE_STORAGE_S3_ACCESS_KEY_ID` | empty | Secret; inject through deployment environment. |
|
||||||
|
| `FILE_STORAGE_S3_SECRET_ACCESS_KEY` | empty | Secret; inject through deployment environment. |
|
||||||
|
| `FILE_STORAGE_S3_BUCKET` | `files` | Managed-file object bucket. |
|
||||||
|
|
||||||
|
Legacy `S3_*` settings remain for older storage paths but new deployments should
|
||||||
|
prefer `FILE_STORAGE_*`.
|
||||||
|
|
||||||
|
### HTTP, Cookies, And Base URLs
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `CORS_ORIGINS` | local dev origins | Set to the exact WebUI origins in staging/production. |
|
||||||
|
| `AUTH_SESSION_COOKIE_NAME` | configured default | Change only through a controlled rollout because it logs users out. |
|
||||||
|
| `AUTH_CSRF_COOKIE_NAME` | configured default | Must match WebUI/API deployment. |
|
||||||
|
| `AUTH_COOKIE_SECURE` | `false` | Set `true` behind HTTPS. |
|
||||||
|
| `AUTH_COOKIE_SAMESITE` | `lax` | Use a stricter value only after testing login and CSRF flows. |
|
||||||
|
| `AUTH_COOKIE_DOMAIN` | empty | Set only when the API and WebUI intentionally share a parent domain. |
|
||||||
|
|
||||||
|
Public URLs are currently supplied by deployment/reverse-proxy configuration and
|
||||||
|
module settings. Do not hardcode them in core; configuration packages should ask
|
||||||
|
for portal, WebUI, postbox, and notification URLs when they become relevant.
|
||||||
|
|
||||||
|
### Module Catalogs, Licenses, And Trust Roots
|
||||||
|
|
||||||
|
| Setting | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` or `GOVOPLAN_MODULE_PACKAGE_CATALOG` | Module package catalog source. |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE` | Preferred production keyring path. |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL` | Approved catalog channel, for example `stable`. |
|
||||||
|
| `GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE` | Trusted license issuer keyring path. |
|
||||||
|
| `GOVOPLAN_LICENSE_ENFORCEMENT` | Enables license enforcement when set to `true`. |
|
||||||
|
|
||||||
|
Trust roots are deployment-managed and should not be editable through the
|
||||||
|
running WebUI.
|
||||||
|
|
||||||
|
### Mail Test Credentials
|
||||||
|
|
||||||
|
Dedicated SMTP/IMAP test credentials belong to the mail/campaign test-bed
|
||||||
|
configuration, not the core runtime contract. Store them in a local ignored
|
||||||
|
`.env` file for the test bed or in CI secrets. Required values are:
|
||||||
|
|
||||||
|
- SMTP host, port, TLS mode, username, password, and envelope/from address.
|
||||||
|
- IMAP host, port, TLS mode, username, password, and append folder.
|
||||||
|
- At least one recipient mailbox that is safe for automated send tests.
|
||||||
|
|
||||||
|
## First Deployment Flow
|
||||||
|
|
||||||
|
1. Create an environment file or secret set with the runtime contract above.
|
||||||
|
2. Install the tagged core and module packages from `requirements-release.txt`.
|
||||||
|
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
||||||
|
artifact from the same release tag.
|
||||||
|
4. Run database migrations with the target `DATABASE_URL`.
|
||||||
|
5. Create the first tenant and system owner through the controlled bootstrap or
|
||||||
|
one-time admin command for the deployment.
|
||||||
|
6. Start the API service with `govoplan_core.server.app:app`.
|
||||||
|
7. Start workers when `CELERY_ENABLED=true`.
|
||||||
|
8. Start the WebUI/reverse proxy and verify CORS/cookie settings.
|
||||||
|
9. Open Admin > System > Modules, verify enabled modules, and save desired
|
||||||
|
module state if it differs from `ENABLED_MODULES`.
|
||||||
|
10. Run health checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS http://127.0.0.1:8000/health
|
||||||
|
curl -fsS http://127.0.0.1:8000/api/v1/platform/modules
|
||||||
|
```
|
||||||
|
|
||||||
|
Authenticated health details require `system:settings:read`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS -H "X-API-Key: $GOVOPLAN_HEALTH_API_KEY" \
|
||||||
|
http://127.0.0.1:8000/health/details
|
||||||
|
```
|
||||||
|
|
||||||
|
## Production-Like Dev Profile
|
||||||
|
|
||||||
|
Use this profile to verify deployment behavior without publishing packages or
|
||||||
|
using real production credentials. The canonical launcher keeps API, worker, and
|
||||||
|
WebUI code in the editable repositories while Docker provides PostgreSQL and
|
||||||
|
Redis:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The helper wrapper provides explicit lifecycle commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/production-like-dev.sh validate-config
|
||||||
|
scripts/production-like-dev.sh seed
|
||||||
|
scripts/production-like-dev.sh start
|
||||||
|
scripts/production-like-dev.sh stop
|
||||||
|
scripts/production-like-dev.sh reset --yes
|
||||||
|
```
|
||||||
|
|
||||||
|
The launcher uses `dev/production-like/.env` when present, otherwise the checked
|
||||||
|
in `.env.example`. It runs:
|
||||||
|
|
||||||
|
- PostgreSQL on `127.0.0.1:55433`
|
||||||
|
- Redis on `127.0.0.1:56379`
|
||||||
|
- explicit `ENABLED_MODULES`
|
||||||
|
- explicit migrations and `--with-dev-data` bootstrap
|
||||||
|
- API via the module-aware devserver
|
||||||
|
- a Celery worker for `send_email,append_sent,default`
|
||||||
|
- WebUI through the Vite dev server
|
||||||
|
- durable local files under `runtime/production-like/files`
|
||||||
|
|
||||||
|
This profile validates explicit migration execution, config loading, module
|
||||||
|
discovery, route aggregation, local storage paths, Redis broker connectivity,
|
||||||
|
worker heartbeats, and health/readiness startup without real production
|
||||||
|
credentials. It does not replace a managed PostgreSQL/Redis/WebUI/worker
|
||||||
|
deployment test.
|
||||||
|
|
||||||
|
To stop PostgreSQL and Redis when the launcher exits:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Module Install/Uninstall Operations
|
||||||
|
|
||||||
|
Use Admin > System > Modules for planning. The running API server validates and
|
||||||
|
queues install plans; it does not run pip/npm or restart itself from an HTTP
|
||||||
|
request. Package mutation belongs to the trusted installer CLI/daemon in an
|
||||||
|
operator shell while maintenance mode is active.
|
||||||
|
|
||||||
|
Preflight from the server shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --format shell
|
||||||
|
```
|
||||||
|
|
||||||
|
Apply a prepared plan directly from a controlled shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --apply --build-webui
|
||||||
|
```
|
||||||
|
|
||||||
|
For production-like runs, prefer supervised mode with migrations, database
|
||||||
|
backup/restore hooks, restart commands, and health checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--supervise \
|
||||||
|
--migrate \
|
||||||
|
--restart-command 'systemctl restart govoplan-api' \
|
||||||
|
--restart-command 'systemctl restart govoplan-worker' \
|
||||||
|
--health-url http://127.0.0.1:8000/health \
|
||||||
|
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
||||||
|
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"'
|
||||||
|
```
|
||||||
|
|
||||||
|
To let the admin UI submit work without executing package managers inside the
|
||||||
|
API process, run the daemon in a separate operator shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--daemon \
|
||||||
|
--migrate \
|
||||||
|
--build-webui \
|
||||||
|
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
||||||
|
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--health-url http://127.0.0.1:8000/health \
|
||||||
|
--restart-command '<restart govoplan server>'
|
||||||
|
```
|
||||||
|
|
||||||
|
The daemon claims one queued request at a time and writes request/run records
|
||||||
|
below `runtime/module-installer`. For process-manager one-shot usage or tests,
|
||||||
|
use `--daemon-once`. Check daemon status with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --daemon-status --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer uses a runtime lock, snapshots `pip freeze` plus WebUI package
|
||||||
|
files, writes a run record, and marks planned rows as applied only after all
|
||||||
|
commands succeed. With `--migrate`, SQLite databases are backed up through
|
||||||
|
SQLite's backup API; non-SQLite databases require
|
||||||
|
`--database-backup-command`, `--database-restore-check-command`, and
|
||||||
|
`--database-restore-command`.
|
||||||
|
|
||||||
|
Database hook commands receive:
|
||||||
|
|
||||||
|
- `GOVOPLAN_INSTALLER_RUN_DIR`
|
||||||
|
- `GOVOPLAN_DATABASE_URL`
|
||||||
|
- `GOVOPLAN_DATABASE_URL_PGTOOLS` for PostgreSQL tools
|
||||||
|
- `GOVOPLAN_DATABASE_BACKUP_PATH`
|
||||||
|
- `GOVOPLAN_DATABASE_BACKUP_METADATA`
|
||||||
|
|
||||||
|
Avoid embedding secrets directly in commands; prefer environment variables,
|
||||||
|
service credentials, or deployment-local secret injection.
|
||||||
|
|
||||||
|
Inspect installer history and lock state from the operator shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --list-runs --format json
|
||||||
|
govoplan-module-installer --show-run <run-id> --format json
|
||||||
|
govoplan-module-installer --lock-status --format json
|
||||||
|
govoplan-module-installer --list-requests --format json
|
||||||
|
govoplan-module-installer --show-request <request-id> --format json
|
||||||
|
govoplan-module-installer --cancel-request <request-id> --format json
|
||||||
|
govoplan-module-installer --retry-request <request-id> --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Rollback uses the saved run snapshot:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --rollback <run-id>
|
||||||
|
govoplan-module-installer --rollback <run-id> --database-restore-command '<override restore command>'
|
||||||
|
```
|
||||||
|
|
||||||
|
Uninstall is non-destructive by default. A planned uninstall row can set
|
||||||
|
`destroy_data: true` to request destructive module retirement. The module must
|
||||||
|
provide an automated retirement provider, and the installer snapshots the
|
||||||
|
database before dropping module-owned tables.
|
||||||
|
|
||||||
|
Run the rollback drill before relying on installer automation in a new
|
||||||
|
environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/module-installer-rollback-drill.py --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The drill uses temporary SQLite databases and simulated package commands. It
|
||||||
|
does not install or uninstall real packages. It exercises:
|
||||||
|
|
||||||
|
- package command failure followed by supervised rollback;
|
||||||
|
- migration failure with a SQLite database snapshot;
|
||||||
|
- restart-command failure;
|
||||||
|
- health timeout after restart;
|
||||||
|
- destructive retirement executor failure with database rollback;
|
||||||
|
- PostgreSQL-style backup, restore-check, and restore hooks;
|
||||||
|
- daemon heartbeat, request queue claim/update, retry/cancel, and stale lock
|
||||||
|
detection/removal.
|
||||||
|
|
||||||
|
See `RELEASE_DEPENDENCIES.md` for release package refs, migration baseline
|
||||||
|
checks, catalog trust, signing, keyring, replay, and license operation.
|
||||||
|
|
||||||
|
## Operator Checklist
|
||||||
|
|
||||||
|
- Runtime secrets are injected outside git.
|
||||||
|
- `MASTER_KEY_B64` is set and backed up securely.
|
||||||
|
- Database backup and restore commands are tested.
|
||||||
|
- File/object storage is durable and backed up.
|
||||||
|
- `CORS_ORIGINS` and cookie settings match the deployed WebUI origin.
|
||||||
|
- Redis and workers are running before `CELERY_ENABLED=true`.
|
||||||
|
- Module catalog and license keyrings are pinned locally.
|
||||||
|
- Health endpoints are monitored.
|
||||||
|
- Test SMTP/IMAP credentials are non-production and isolated.
|
||||||
|
- Module installer rollback drill has passed in the deployment environment.
|
||||||
52
docs/DOCUMENTATION_MAP.md
Normal file
52
docs/DOCUMENTATION_MAP.md
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
# GovOPlaN Documentation Map
|
||||||
|
|
||||||
|
This map defines the source-of-truth documents for the current repository docs.
|
||||||
|
Use it to avoid duplicating long procedures across architecture, release,
|
||||||
|
operator, and roadmap pages.
|
||||||
|
|
||||||
|
## Core Platform
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Module architecture and kernel contracts | `MODULE_ARCHITECTURE.md` | Stable module contracts, API efficiency contracts, durable boundary decisions, lifecycle, and WebUI contribution rules. |
|
||||||
|
| RBAC and resource access | `ACCESS_RBAC_MODEL.md` | Current permission, role, API-key, and resource-access model. |
|
||||||
|
| Governance hierarchy | `GOVERNANCE_MODEL.md` | System, tenant, user/group, campaign policy inheritance and admin UI structure. |
|
||||||
|
| Policy decision DTOs and provenance | `POLICY_CONTRACTS.md` | Shared explain/provenance shape; module-specific policy docs should link here. |
|
||||||
|
| Events and audit trace context | `EVENTS_AND_AUDIT.md` | Event dispatch semantics and audit payload conventions. |
|
||||||
|
| Action/effect automation layer | `ACTION_EFFECT_AUTOMATION_LAYER.md` | Action/effect contracts, consequence preview, runner semantics, and module boundary for automation. |
|
||||||
|
| Postbox E2EE target architecture | `POSTBOX_E2EE_ARCHITECTURE.md` | Strategic encrypted postbox/mailbox model, key ownership, role mailbox semantics, and retraction limits. |
|
||||||
|
|
||||||
|
## Release And Operations
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Runtime configuration and operator flow | `DEPLOYMENT_OPERATOR_GUIDE.md` | Production/staging configuration, migrations, backups, installer operation, and rollback drill. |
|
||||||
|
| Self-hosted installability | `SELF_HOSTED_INSTALLABILITY.md` | Packaging decision, generated env templates, config validation, production-like dev stack commands, and boundary gate. |
|
||||||
|
| Release dependencies and catalogs | `RELEASE_DEPENDENCIES.md` | Release package refs, migration baselines, release lockfiles, catalog trust/licensing, catalog publishing, and release checklist. |
|
||||||
|
| Dependency vulnerability audits | `DEPENDENCY_AUDITS.md` | Local and CI audit commands plus dated audit result notes. |
|
||||||
|
| Remote WebUI bundle design | `REMOTE_WEBUI_BUNDLES.md` | Experimental controlled-deployment design; normal releases still use package builds. |
|
||||||
|
|
||||||
|
## Product And Module Planning
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Product roadmap and module routing | `GOVOPLAN_MASTER_ROADMAP.md` | Product-level sequencing, implementation gates, issue routing, and missing-module decisions. |
|
||||||
|
| UI/UX decisions | `UI_UX_DECISION_LEDGER.md` | Binding guided-UI decisions, open decisions, impact index, and review checklist. |
|
||||||
|
| Interface ethics and design doctrine | `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` | Product-level doctrine for context, decision, consequence, contestability, responsibility, and traceability. |
|
||||||
|
| Public-sector integration posture | `PUBLIC_SECTOR_INTEGRATION_STRATEGY.md` | Strategy index; executable target inventory lives in `govoplan-connectors`. |
|
||||||
|
| Configuration packages | `CONFIGURATION_PACKAGES.md` | Package model, provider contract, import/export flow, and tracking slices. |
|
||||||
|
|
||||||
|
## Workflow Docs
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Gitea issues and wiki sync | `GITEA_ISSUES.md` | Issue labels, imports, wiki mirroring, and Codex state updates. |
|
||||||
|
| Codex local workflow | `CODEX_WORKFLOW.md` | Local agent setup and focused verification commands. |
|
||||||
|
|
||||||
|
## Cross-Repo Rule
|
||||||
|
|
||||||
|
Core docs may keep strategy, kernel contracts, and routing decisions. Module
|
||||||
|
repositories should own executable module behavior, concrete API/UI contracts,
|
||||||
|
and operator notes for their own module. When content spans both, keep the
|
||||||
|
durable decision in core and link to the module document for implementation
|
||||||
|
details.
|
||||||
203
docs/EVENTS_AND_AUDIT.md
Normal file
203
docs/EVENTS_AND_AUDIT.md
Normal file
@@ -0,0 +1,203 @@
|
|||||||
|
# Events And Audit
|
||||||
|
|
||||||
|
GovOPlaN uses a small kernel event contract first, not a broad command bus.
|
||||||
|
Commands remain module-owned application-service methods or API endpoints until
|
||||||
|
there is a concrete need for durable asynchronous command orchestration. Events
|
||||||
|
are facts about completed work and are safe for audit, projections, optional
|
||||||
|
module reactions, and operator diagnostics.
|
||||||
|
|
||||||
|
## Production Transport Decision
|
||||||
|
|
||||||
|
The first production target is a **database outbox plus in-process immediate
|
||||||
|
dispatch**:
|
||||||
|
|
||||||
|
- Use `govoplan_core.core.events.PlatformEvent` for domain and platform events.
|
||||||
|
- Use `EventBus` as the in-process dispatch contract for same-process module
|
||||||
|
reactions that are safe to run inline.
|
||||||
|
- Use the shared `audit_event` / `audit_from_principal` helper for audited
|
||||||
|
module actions. The helper persists the audit row and immediately publishes a
|
||||||
|
governed `PlatformEvent` whose `type` is the audit action.
|
||||||
|
- Use `record_change` for module delta feeds. It persists the change-sequence
|
||||||
|
row and immediately publishes a generic module change event such as
|
||||||
|
`mail.profile.updated`.
|
||||||
|
- Persist durable integration/workflow events through a database outbox before
|
||||||
|
acknowledging the state change that produced them.
|
||||||
|
- Drain the outbox through a small dispatcher process. The dispatcher may call
|
||||||
|
in-process handlers in the same deployment first, but its storage contract is
|
||||||
|
database-backed.
|
||||||
|
- Treat Redis/Celery as worker/job infrastructure, not as the authoritative
|
||||||
|
first event transport. A Celery dispatcher can consume the outbox later.
|
||||||
|
- Keep the dispatch implementation pluggable behind the `PlatformEvent`
|
||||||
|
envelope so a future message broker can be added without changing event
|
||||||
|
producers.
|
||||||
|
- Keep commands out of the kernel until workflows need retryable, durable,
|
||||||
|
operator-visible command records.
|
||||||
|
|
||||||
|
This keeps the first contract small, PostgreSQL-friendly, auditable, and
|
||||||
|
recoverable after process crashes. It also avoids making Redis a correctness
|
||||||
|
dependency for deployments that only need synchronous mail/tests or light
|
||||||
|
background work.
|
||||||
|
|
||||||
|
## Dispatch Semantics
|
||||||
|
|
||||||
|
Event producers should write their domain state and outbox event in the same
|
||||||
|
database transaction wherever possible. Handlers must be idempotent because the
|
||||||
|
outbox dispatcher can retry after a crash or timeout.
|
||||||
|
|
||||||
|
Recommended first outbox columns:
|
||||||
|
|
||||||
|
- `event_id`, `event_type`, `module_id`
|
||||||
|
- `correlation_id`, `causation_id`
|
||||||
|
- `payload`, `occurred_at`
|
||||||
|
- `available_at`, `attempt_count`, `claimed_at`, `claim_token`
|
||||||
|
- `processed_at`, `last_error`
|
||||||
|
|
||||||
|
Inline `EventBus` handlers are allowed only for non-critical local reactions.
|
||||||
|
Anything that must survive process failure, restart, package update, or worker
|
||||||
|
redeployment belongs in the outbox.
|
||||||
|
|
||||||
|
## Trace IDs
|
||||||
|
|
||||||
|
Every `PlatformEvent` has:
|
||||||
|
|
||||||
|
- `event_id`: unique ID for that event.
|
||||||
|
- `correlation_id`: stable ID for the whole request, workflow, or job.
|
||||||
|
- `causation_id`: the event ID or external operation ID that caused this
|
||||||
|
event.
|
||||||
|
- `actor`: optional typed actor reference, for example user, API key, system
|
||||||
|
actor, delegated actor, or installer daemon.
|
||||||
|
- `tenant`: optional tenant reference when the event is tenant-scoped.
|
||||||
|
- `subject`: optional typed subject reference for the person, organization,
|
||||||
|
account, case, campaign, or other entity the event is about.
|
||||||
|
- `resource`: optional typed resource reference for the object changed or
|
||||||
|
observed by the event.
|
||||||
|
- `classification`: payload sensitivity, currently `public`, `internal`,
|
||||||
|
`confidential`, or `restricted`.
|
||||||
|
- `payload`: JSON-serializable module-owned event details.
|
||||||
|
|
||||||
|
The FastAPI app factory creates an event context for every request. It accepts
|
||||||
|
`X-Correlation-ID` or `X-Request-ID` when the value is a compact safe trace ID,
|
||||||
|
otherwise it generates a new ID. Responses include `X-Correlation-ID`.
|
||||||
|
|
||||||
|
Audit logging reads the current event context and stores trace IDs in
|
||||||
|
`details._trace`. Callers can also pass explicit `correlation_id` and
|
||||||
|
`causation_id` to `audit_event` or `audit_from_principal`. The same trace is
|
||||||
|
copied into the emitted `PlatformEvent`, so audit rows and event subscribers can
|
||||||
|
be correlated without route-specific glue code.
|
||||||
|
|
||||||
|
Admin and lifecycle code should use the compact operational detail shape
|
||||||
|
documented in `govoplan-audit/docs/AUDIT_TRACE_CONTEXT.md`. The core
|
||||||
|
`audit_operation_context` helper preserves `module_id`, `request_id`, `run_id`,
|
||||||
|
`outcome`, and `_trace` while applying the shared audit redaction pass to
|
||||||
|
additional detail values.
|
||||||
|
|
||||||
|
## Audit MVP Boundary
|
||||||
|
|
||||||
|
`govoplan-audit` owns:
|
||||||
|
|
||||||
|
- the `audit_log` table and audit API routes
|
||||||
|
- audit route contribution through its module manifest
|
||||||
|
- audit retention behavior in cooperation with policy/retention settings
|
||||||
|
- future audit sink/export capability implementations
|
||||||
|
|
||||||
|
`govoplan-core` owns:
|
||||||
|
|
||||||
|
- `AuditEvent` and `AuditSink` protocol contracts
|
||||||
|
- request and event trace context
|
||||||
|
- the compatibility `audit_event` helper while routes are still migrating
|
||||||
|
- retention orchestration that calls module capabilities
|
||||||
|
|
||||||
|
Feature modules should record audit facts through the shared helper or a future
|
||||||
|
`audit.sink` capability. They should not import audit storage internals. Module
|
||||||
|
state changes that only need delta-feed visibility should go through
|
||||||
|
`record_change`; route-level business actions should still record a semantic
|
||||||
|
audit action.
|
||||||
|
|
||||||
|
## Initial Domain Event Inventory
|
||||||
|
|
||||||
|
Access:
|
||||||
|
|
||||||
|
- `access.account.created`
|
||||||
|
- `access.account.updated`
|
||||||
|
- `access.membership.created`
|
||||||
|
- `access.membership.updated`
|
||||||
|
- `access.group.created`
|
||||||
|
- `access.group.updated`
|
||||||
|
- `access.role.created`
|
||||||
|
- `access.role.updated`
|
||||||
|
- `access.role.deleted`
|
||||||
|
- `access.api_key.created`
|
||||||
|
- `access.api_key.revoked`
|
||||||
|
- `access.session.created`
|
||||||
|
- `access.session.revoked`
|
||||||
|
|
||||||
|
Tenancy:
|
||||||
|
|
||||||
|
- `tenant.created`
|
||||||
|
- `tenant.updated`
|
||||||
|
- `tenant.suspended`
|
||||||
|
- `tenant.resumed`
|
||||||
|
- `tenant.deletion_requested`
|
||||||
|
- `tenant.erasure_completed`
|
||||||
|
|
||||||
|
Policy:
|
||||||
|
|
||||||
|
- `policy.system.updated`
|
||||||
|
- `policy.tenant.updated`
|
||||||
|
- `policy.user.updated`
|
||||||
|
- `policy.group.updated`
|
||||||
|
- `policy.campaign.updated`
|
||||||
|
- `policy.effective_policy.changed`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `files.file.uploaded`
|
||||||
|
- `files.file.renamed`
|
||||||
|
- `files.file.deleted`
|
||||||
|
- `files.file.frozen`
|
||||||
|
- `files.share.created`
|
||||||
|
- `files.share.revoked`
|
||||||
|
- `files.connector.imported`
|
||||||
|
- `files.connector.access_denied`
|
||||||
|
|
||||||
|
Mail:
|
||||||
|
|
||||||
|
- `mail.profile.created`
|
||||||
|
- `mail.profile.updated`
|
||||||
|
- `mail.profile.credentials_rotated`
|
||||||
|
- `mail.profile.tested`
|
||||||
|
- `mail.message.sent`
|
||||||
|
- `mail.message.send_failed`
|
||||||
|
- `mail.imap.appended`
|
||||||
|
- `mail.imap.append_failed`
|
||||||
|
- `mail.mailbox.message_seen`
|
||||||
|
|
||||||
|
Campaign:
|
||||||
|
|
||||||
|
- `campaign.created`
|
||||||
|
- `campaign.version.created`
|
||||||
|
- `campaign.validated`
|
||||||
|
- `campaign.built`
|
||||||
|
- `campaign.reviewed`
|
||||||
|
- `campaign.queued`
|
||||||
|
- `campaign.send_started`
|
||||||
|
- `campaign.recipient_attempted`
|
||||||
|
- `campaign.recipient_delivered`
|
||||||
|
- `campaign.recipient_failed`
|
||||||
|
- `campaign.paused`
|
||||||
|
- `campaign.resumed`
|
||||||
|
- `campaign.cancelled`
|
||||||
|
- `campaign.report.exported`
|
||||||
|
|
||||||
|
## Event Payload Rules
|
||||||
|
|
||||||
|
- Payloads must be JSON-serializable.
|
||||||
|
- Use stable IDs, not ORM objects.
|
||||||
|
- Include tenant ID when tenant-scoped.
|
||||||
|
- Include actor/principal, subject, and resource references only as typed DTOs
|
||||||
|
or primitive IDs.
|
||||||
|
- Set `classification` to the highest sensitivity needed by the envelope or
|
||||||
|
payload, not the lowest sensitivity of any individual field.
|
||||||
|
- Do not include secrets, raw message bodies, full recipient lists, or file
|
||||||
|
content.
|
||||||
|
- Put large evidence in owning module storage and reference it by ID.
|
||||||
319
docs/GITEA_ISSUES.md
Normal file
319
docs/GITEA_ISSUES.md
Normal file
@@ -0,0 +1,319 @@
|
|||||||
|
# Gitea Issues And Wiki Workflow
|
||||||
|
|
||||||
|
Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes.
|
||||||
|
|
||||||
|
The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state.
|
||||||
|
|
||||||
|
## Initial Setup
|
||||||
|
|
||||||
|
The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`.
|
||||||
|
|
||||||
|
The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan-core.git`). Override inference when needed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export GITEA_URL=https://git.add-ideas.de
|
||||||
|
export GITEA_OWNER=add-ideas
|
||||||
|
export GITEA_REPO=govoplan-core
|
||||||
|
export GITEA_TOKEN=...
|
||||||
|
```
|
||||||
|
|
||||||
|
The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GITEA_TOKEN=...
|
||||||
|
# Optional if origin inference is not enough:
|
||||||
|
GITEA_URL=https://git.add-ideas.de
|
||||||
|
GITEA_OWNER=add-ideas
|
||||||
|
GITEA_REPO=govoplan-core
|
||||||
|
```
|
||||||
|
|
||||||
|
For a shared credentials file outside the target repository, pass `--env-file`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a Gitea token with issue read/write access and label-management
|
||||||
|
permission for the repository. On scoped-token instances, this usually means
|
||||||
|
issue read/write and, if label writes are rejected, repository write permission
|
||||||
|
too.
|
||||||
|
|
||||||
|
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
|
||||||
|
Creating or updating organization labels requires a token with
|
||||||
|
`write:organization`. Repository label management only needs repository label
|
||||||
|
permission, but it duplicates the taxonomy into each repository.
|
||||||
|
|
||||||
|
Preview and apply labels:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-sync-labels.py
|
||||||
|
./scripts/gitea-sync-labels.py --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Preview and apply the shared taxonomy as organization labels:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
|
||||||
|
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The import helpers resolve repository labels and organization labels. Repository
|
||||||
|
labels win when a repository defines the same name locally, but a repository does
|
||||||
|
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
|
||||||
|
`module/*`, `area/*`, `source/*`, or `codex/*` label.
|
||||||
|
|
||||||
|
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
||||||
|
|
||||||
|
## Multiple Repositories And Workspaces
|
||||||
|
|
||||||
|
The helper scripts are path-based and can run from this core checkout against any repository with a Gitea remote:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply
|
||||||
|
./scripts/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail
|
||||||
|
./scripts/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress
|
||||||
|
```
|
||||||
|
|
||||||
|
Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH.
|
||||||
|
|
||||||
|
When one core checkout drives another workspace, prefer `--env-file` for shared credentials instead of putting `GITEA_REPO` in the core `.env`; a repo-specific `GITEA_REPO` can accidentally override target inference.
|
||||||
|
|
||||||
|
For non-GovOPlaN projects, provide a project-specific label file and module/project label:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-labels.py \
|
||||||
|
--root /path/to/project \
|
||||||
|
--labels-file /path/to/project/docs/gitea-labels.json \
|
||||||
|
--apply
|
||||||
|
|
||||||
|
./scripts/gitea-todo-import.py \
|
||||||
|
--root /path/to/project \
|
||||||
|
--module-label project/example \
|
||||||
|
--extra-label area/backend
|
||||||
|
```
|
||||||
|
|
||||||
|
If another project does not use `area/*` labels, disable area inference:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-todo-import.py \
|
||||||
|
--root /path/to/project \
|
||||||
|
--module-label project/example \
|
||||||
|
--no-area-labels
|
||||||
|
```
|
||||||
|
|
||||||
|
Install or refresh the shared issue templates in sibling or external repositories:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail
|
||||||
|
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-<name>` checkout maps to `module/<name>`. For another workspace or repository name, pass an explicit label:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-install-workflow.py /path/to/repo --module-label module/example --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in core and run the sync script from core.
|
||||||
|
|
||||||
|
For a fully portable workflow kit, copy these files into the other project:
|
||||||
|
|
||||||
|
- `scripts/gitea_common.py`
|
||||||
|
- `scripts/gitea-sync-labels.py`
|
||||||
|
- `scripts/gitea-todo-import.py`
|
||||||
|
- `scripts/gitea-codex-note.py`
|
||||||
|
- `scripts/gitea-install-workflow.py`
|
||||||
|
- `.gitea/ISSUE_TEMPLATE/*`
|
||||||
|
- `.gitea/PULL_REQUEST_TEMPLATE.md`
|
||||||
|
- a project-specific `docs/gitea-labels.json`
|
||||||
|
|
||||||
|
Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup.
|
||||||
|
|
||||||
|
## Label Taxonomy
|
||||||
|
|
||||||
|
Use one `type/*` label:
|
||||||
|
|
||||||
|
- `type/bug`
|
||||||
|
- `type/feature`
|
||||||
|
- `type/task`
|
||||||
|
- `type/debt`
|
||||||
|
- `type/docs`
|
||||||
|
|
||||||
|
Use one `status/*` label while the issue is open:
|
||||||
|
|
||||||
|
- `status/triage`: needs ownership, priority, or acceptance criteria.
|
||||||
|
- `status/ready`: ready to implement.
|
||||||
|
- `status/in-progress`: actively being worked.
|
||||||
|
- `status/blocked`: blocked on an external dependency, credential, or decision.
|
||||||
|
- `status/needs-info`: blocked on clarification.
|
||||||
|
|
||||||
|
Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`.
|
||||||
|
|
||||||
|
Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository.
|
||||||
|
|
||||||
|
Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first.
|
||||||
|
|
||||||
|
## Moving TODOs Into Gitea
|
||||||
|
|
||||||
|
Preview inline markers:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-todo-import.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Create missing issues after labels are synced:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-todo-import.py --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items.
|
||||||
|
|
||||||
|
When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# TODO(gitea#123): keep only if the local pointer is still useful
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers.
|
||||||
|
|
||||||
|
For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||||
|
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
It scans repository and product-directory files with backlog-like names, resolves
|
||||||
|
shared labels from the organization label catalogue where available, creates only
|
||||||
|
missing fallback repository labels when needed, imports missing open work, and
|
||||||
|
deduplicates reruns by hidden fingerprint and normalized title.
|
||||||
|
|
||||||
|
## Mirroring Project Docs Into Gitea Wikis
|
||||||
|
|
||||||
|
Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||||
|
```
|
||||||
|
|
||||||
|
Apply the wiki mirror:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
After renaming or deleting repository docs, prune previously managed wiki pages
|
||||||
|
that no longer have a source file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The default apply path uses the Gitea wiki git repository, not one REST API
|
||||||
|
request per page. It keeps a local checkout cache below
|
||||||
|
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
||||||
|
pushes that commit. This is much faster and avoids REST wiki-page timeouts.
|
||||||
|
Use `--transport api` only when the wiki git remote is unavailable.
|
||||||
|
|
||||||
|
Limit a sync to one repository or one generated page while working:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-wiki.py --repo govoplan-core --apply
|
||||||
|
./scripts/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository
|
||||||
|
sync when the set of mirrored pages changes.
|
||||||
|
|
||||||
|
The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related.
|
||||||
|
|
||||||
|
Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages.
|
||||||
|
|
||||||
|
Use the wiki for durable context:
|
||||||
|
|
||||||
|
- project overviews and architecture
|
||||||
|
- workflows, operating notes, and setup references
|
||||||
|
- product concepts, plans, pitches, and whitepapers
|
||||||
|
- historical context that helps interpret issues
|
||||||
|
|
||||||
|
Keep active state in issues:
|
||||||
|
|
||||||
|
- open tasks, TODOs, feature requests, and bugs
|
||||||
|
- priority, blocking status, and acceptance criteria
|
||||||
|
- Codex progress updates and implementation notes
|
||||||
|
|
||||||
|
## Codex State Updates
|
||||||
|
|
||||||
|
Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes:
|
||||||
|
|
||||||
|
- scope understood
|
||||||
|
- files changed
|
||||||
|
- tests or manual checks run
|
||||||
|
- blockers or decisions needed
|
||||||
|
- follow-up issues created
|
||||||
|
|
||||||
|
Preview and post a standardized note:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-codex-note.py \
|
||||||
|
--issue 123 \
|
||||||
|
--status progress \
|
||||||
|
--summary "Implemented capability metadata fallback." \
|
||||||
|
--changed src/govoplan_core/modules/registry.py \
|
||||||
|
--test "./.venv/bin/python -m unittest tests.test_module_system"
|
||||||
|
|
||||||
|
./scripts/gitea-codex-note.py \
|
||||||
|
--issue 123 \
|
||||||
|
--status progress \
|
||||||
|
--summary "Implemented capability metadata fallback." \
|
||||||
|
--changed src/govoplan_core/modules/registry.py \
|
||||||
|
--test "./.venv/bin/python -m unittest tests.test_module_system" \
|
||||||
|
--apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded.
|
||||||
|
|
||||||
|
## Ownership Rules
|
||||||
|
|
||||||
|
Create the issue in the repository that owns the change:
|
||||||
|
|
||||||
|
- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components.
|
||||||
|
- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration.
|
||||||
|
- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations.
|
||||||
|
- `govoplan-files`: files-specific backend, frontend, storage, and file workflows.
|
||||||
|
- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior.
|
||||||
|
|
||||||
|
For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details.
|
||||||
|
|
||||||
|
## Cleaning Up Mirrored Sources
|
||||||
|
|
||||||
|
After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state.
|
||||||
|
|
||||||
|
Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding.
|
||||||
|
|
||||||
|
Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path.
|
||||||
|
|
||||||
|
## Docs Versus Issues
|
||||||
|
|
||||||
|
Keep durable facts in docs:
|
||||||
|
|
||||||
|
- architecture and extension points
|
||||||
|
- command references
|
||||||
|
- module boundaries
|
||||||
|
- operational conventions
|
||||||
|
|
||||||
|
Keep changing state in Gitea:
|
||||||
|
|
||||||
|
- TODOs and follow-ups
|
||||||
|
- bugs and feature requests
|
||||||
|
- blocked status
|
||||||
|
- acceptance criteria
|
||||||
|
- implementation notes from active work
|
||||||
|
|
||||||
|
If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history.
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
# Multi Seal Mail - Current System and Tenant Governance Model
|
# GovOPlaN Governance Model
|
||||||
|
|
||||||
**Updated:** 2026-06-16
|
**Updated:** 2026-07-09
|
||||||
**Current migration head:** `f5a6b7c8d9e0`
|
|
||||||
|
|
||||||
## Governance Rule
|
## Governance Rule
|
||||||
|
|
||||||
System policy is authoritative for tenants and all lower levels. Each lower level may only narrow what it inherits:
|
System policy is authoritative for tenants and all lower levels. Each lower
|
||||||
|
level may only narrow what it inherits:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
system
|
system
|
||||||
@@ -14,52 +14,65 @@ system
|
|||||||
-> campaign
|
-> campaign
|
||||||
```
|
```
|
||||||
|
|
||||||
Lower levels do not widen privileges, allowed profiles, retention durations or credential rights granted by a higher level.
|
Lower levels do not widen privileges, allowed profiles, retention durations, or
|
||||||
|
credential rights granted by a higher level.
|
||||||
|
|
||||||
## Administration Structure
|
## Administration Structure
|
||||||
|
|
||||||
|
GovOPlaN separates system administration from scoped configuration:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
SYSTEM
|
ADMINISTRATION
|
||||||
- Settings
|
- Modules
|
||||||
- Retention
|
- Packages
|
||||||
- Mail servers
|
- Maintenance
|
||||||
|
- Changes
|
||||||
|
|
||||||
|
GLOBAL
|
||||||
- Tenants
|
- Tenants
|
||||||
- Users
|
- Roles
|
||||||
- Groups
|
- Groups and users
|
||||||
- System roles
|
- File connectors
|
||||||
- Tenant roles
|
- Mail servers
|
||||||
- Audit
|
- API keys
|
||||||
|
- Retention
|
||||||
|
|
||||||
TENANT
|
TENANT
|
||||||
- Settings boundary
|
|
||||||
- Users
|
|
||||||
- Groups
|
|
||||||
- Roles
|
- Roles
|
||||||
- API keys
|
- Groups and users
|
||||||
|
- File connectors
|
||||||
- Mail servers
|
- Mail servers
|
||||||
|
- API keys
|
||||||
- Retention
|
- Retention
|
||||||
- Audit
|
|
||||||
|
|
||||||
USER
|
|
||||||
- User mail
|
|
||||||
- User retention
|
|
||||||
|
|
||||||
GROUP
|
GROUP
|
||||||
- Group mail
|
- File connectors
|
||||||
- Group retention
|
- Mail servers
|
||||||
|
- API keys
|
||||||
|
- Retention
|
||||||
|
|
||||||
|
USER
|
||||||
|
- File connectors
|
||||||
|
- Mail servers
|
||||||
|
- API keys
|
||||||
|
- Retention
|
||||||
```
|
```
|
||||||
|
|
||||||
There is no separate System access page. Compatibility access scopes remain in the backend for assignment/read boundaries.
|
System access scopes remain in the backend for assignment/read boundaries, but
|
||||||
|
the UI should present the configuration hierarchy rather than a separate
|
||||||
|
"system access" concept.
|
||||||
|
|
||||||
## Tenant Governance
|
## Tenant Governance
|
||||||
|
|
||||||
System settings define tenant defaults and whether tenants may narrow selected options. Tenant overrides can only restrict:
|
System settings define tenant defaults and whether tenants may narrow selected
|
||||||
|
options. Tenant overrides can only restrict:
|
||||||
|
|
||||||
- custom groups;
|
- custom groups;
|
||||||
- custom roles;
|
- custom roles;
|
||||||
- tenant API keys.
|
- tenant API keys.
|
||||||
|
|
||||||
The backend enforces that tenant governance cannot widen system-denied privileges.
|
The backend enforces that tenant governance cannot widen system-denied
|
||||||
|
privileges.
|
||||||
|
|
||||||
## Mail-Profile Governance
|
## Mail-Profile Governance
|
||||||
|
|
||||||
@@ -73,7 +86,10 @@ group
|
|||||||
campaign
|
campaign
|
||||||
```
|
```
|
||||||
|
|
||||||
Effective campaign profile availability follows campaign ownership. A campaign owned by a user resolves through system, tenant, that user and campaign policy. A group-owned campaign resolves through system, tenant, that group and campaign policy.
|
Effective campaign profile availability follows campaign ownership. A campaign
|
||||||
|
owned by a user resolves through system, tenant, that user, and campaign
|
||||||
|
policy. A group-owned campaign resolves through system, tenant, that group, and
|
||||||
|
campaign policy.
|
||||||
|
|
||||||
Policy semantics:
|
Policy semantics:
|
||||||
|
|
||||||
@@ -81,12 +97,18 @@ Policy semantics:
|
|||||||
- lower levels can further restrict the set;
|
- lower levels can further restrict the set;
|
||||||
- forced profiles mean the lower level must choose from the forced set;
|
- forced profiles mean the lower level must choose from the forced set;
|
||||||
- a forced set with one profile effectively enforces that profile;
|
- a forced set with one profile effectively enforces that profile;
|
||||||
- campaign-level profile creation is allowed only if the effective policy permits it;
|
- campaign-level profile creation is allowed only if the effective policy
|
||||||
- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels must inherit profile credentials, may inherit profile credentials, or must provide local credentials;
|
permits it;
|
||||||
- the lower-level override switch for `smtp_credentials.inherit` and `imap_credentials.inherit` controls whether descendants may change that inheritance decision;
|
- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels
|
||||||
|
must inherit profile credentials, may inherit profile credentials, or must
|
||||||
|
provide local credentials;
|
||||||
|
- the lower-level override switch for `smtp_credentials.inherit` and
|
||||||
|
`imap_credentials.inherit` controls whether descendants may change that
|
||||||
|
inheritance decision;
|
||||||
- deny patterns always win over allow patterns;
|
- deny patterns always win over allow patterns;
|
||||||
- empty or `*` allowlist means allow all except denied;
|
- empty or `*` allowlist means allow all except denied;
|
||||||
- non-empty allowlist means at least one allow rule must match and no deny rule may match.
|
- non-empty allowlist means at least one allow rule must match and no deny rule
|
||||||
|
may match.
|
||||||
|
|
||||||
Pattern targets:
|
Pattern targets:
|
||||||
|
|
||||||
@@ -98,7 +120,23 @@ From header
|
|||||||
recipient domains
|
recipient domains
|
||||||
```
|
```
|
||||||
|
|
||||||
Ownership transfer is intentionally deferred as a two-step workflow: original owner initiates, new owner accepts and reselects/repairs the mail profile if their effective policy requires it.
|
Ownership transfer is intentionally deferred as a two-step workflow: original
|
||||||
|
owner initiates, new owner accepts and reselects/repairs the mail profile if
|
||||||
|
their effective policy requires it.
|
||||||
|
|
||||||
|
## File-Connector Governance
|
||||||
|
|
||||||
|
File connector profiles and credentials are separated. Profiles describe
|
||||||
|
external endpoints; credentials bind authentication material and policy to a
|
||||||
|
scope. Concrete linked folders appear as file spaces in the files module.
|
||||||
|
|
||||||
|
Governance follows the same inheritance shape as mail:
|
||||||
|
|
||||||
|
- system and tenant policy can permit, require, or forbid lower-level
|
||||||
|
connections/credentials;
|
||||||
|
- user or group ownership controls which spaces appear to principals;
|
||||||
|
- spaces inherit endpoint and credential policy from their connector profile;
|
||||||
|
- connector health and credential tests must not reveal plaintext secrets.
|
||||||
|
|
||||||
## Retention Governance
|
## Retention Governance
|
||||||
|
|
||||||
@@ -120,20 +158,27 @@ Rules:
|
|||||||
|
|
||||||
- system may set concrete defaults or unlimited retention;
|
- system may set concrete defaults or unlimited retention;
|
||||||
- system exposes allow-limiting toggles per field;
|
- system exposes allow-limiting toggles per field;
|
||||||
- tenants, users/groups and campaigns may only shorten inherited retention where the parent allows limiting;
|
- tenants, users/groups, and campaigns may only shorten inherited retention
|
||||||
|
where the parent allows limiting;
|
||||||
- blank lower-level values inherit;
|
- blank lower-level values inherit;
|
||||||
- mock mailbox retention is currently system-level because mock mailbox records do not yet carry tenant/campaign ownership metadata;
|
- mock mailbox retention is currently system-level because mock mailbox records
|
||||||
- dry-run/apply retention actions report affected classes before destructive cleanup.
|
do not yet carry tenant/campaign ownership metadata;
|
||||||
|
- dry-run/apply retention actions report affected classes before destructive
|
||||||
|
cleanup.
|
||||||
|
|
||||||
## Role Definitions and Assignments
|
## Role Definitions And Assignments
|
||||||
|
|
||||||
### System roles
|
### System Roles
|
||||||
|
|
||||||
System roles define instance-wide permissions. `system:*` is stored as one wildcard and displayed as granting the full system catalogue. System owner is protected.
|
System roles define instance-wide permissions. `system:*` is stored as one
|
||||||
|
wildcard and displayed as granting the full system catalogue. System owner is
|
||||||
|
protected.
|
||||||
|
|
||||||
### Tenant roles
|
### Tenant Roles
|
||||||
|
|
||||||
Tenant roles can be system-governed templates or tenant-local definitions, subject to system tenant-governance settings and actor delegation ceilings. Wildcard counts are expanded against the canonical tenant catalogue.
|
Tenant roles can be system-governed templates or tenant-local definitions,
|
||||||
|
subject to system tenant-governance settings and actor delegation ceilings.
|
||||||
|
Wildcard counts are expanded against the canonical tenant catalogue.
|
||||||
|
|
||||||
## Audit Access
|
## Audit Access
|
||||||
|
|
||||||
@@ -144,15 +189,17 @@ system audit -> system:audit:read
|
|||||||
tenant audit -> active tenant + audit:read
|
tenant audit -> active tenant + audit:read
|
||||||
```
|
```
|
||||||
|
|
||||||
Audit pages use server pagination, filtering and bounded grids.
|
Audit pages use server pagination, filtering, and bounded grids.
|
||||||
|
|
||||||
## Tenant Switching
|
## Tenant Switching
|
||||||
|
|
||||||
Tenant switching preserves the current URL when possible and falls back when a route/resource is not accessible in the new tenant context.
|
Tenant switching preserves the current URL when possible and falls back when a
|
||||||
|
route/resource is not accessible in the new tenant context.
|
||||||
|
|
||||||
The tenant selector is hidden for ordinary single-tenant accounts and visible for multi-tenant or system tenant-management contexts.
|
The tenant selector is hidden for ordinary single-tenant accounts and visible
|
||||||
|
for multi-tenant or system tenant-management contexts.
|
||||||
|
|
||||||
## DataGrid Contract in Administration
|
## Administration DataGrid Contract
|
||||||
|
|
||||||
Admin lists use bounded container grids:
|
Admin lists use bounded container grids:
|
||||||
|
|
||||||
@@ -164,14 +211,16 @@ Admin lists use bounded container grids:
|
|||||||
- sticky headers where needed;
|
- sticky headers where needed;
|
||||||
- server pagination for audit.
|
- server pagination for audit.
|
||||||
|
|
||||||
## Still Deferred
|
## Deferred Work
|
||||||
|
|
||||||
|
- DataGrid sizing and resize behavior remains explicitly deferred. The current
|
||||||
|
bounded-grid contract above is binding, but further layout changes should be
|
||||||
|
handled as a dedicated, isolated UI debt item because the component is shared
|
||||||
|
and brittle.
|
||||||
- real SMTP/IMAP test-bed verification and operator runbook;
|
- real SMTP/IMAP test-bed verification and operator runbook;
|
||||||
- recipient import with column mapping;
|
- recipient import with column mapping;
|
||||||
- Seafile/external connector governance;
|
|
||||||
- system/tenant/group/user file-space hierarchy and external storage hierarchy;
|
|
||||||
- session/device revocation UI;
|
- session/device revocation UI;
|
||||||
- backup/restore, monitoring and update procedures;
|
- backup/restore, monitoring, and update procedures;
|
||||||
- DSAR workflows and evidence bundle verifier;
|
- DSAR workflows and evidence bundle verifier;
|
||||||
- campaign ownership transfer workflow;
|
- campaign ownership transfer workflow;
|
||||||
- policy impact analysis before delete/disable/unshare/change;
|
- policy impact analysis before delete/disable/unshare/change;
|
||||||
663
docs/GOVOPLAN_MASTER_ROADMAP.md
Normal file
663
docs/GOVOPLAN_MASTER_ROADMAP.md
Normal file
@@ -0,0 +1,663 @@
|
|||||||
|
# GovOPlaN Master Roadmap
|
||||||
|
|
||||||
|
This roadmap is the durable product north star and sequencing guide for
|
||||||
|
GovOPlaN as a modular platform for administrative operations. It keeps the
|
||||||
|
product moving without turning every possible public-sector need into an
|
||||||
|
immediate implementation track.
|
||||||
|
|
||||||
|
Use this document for product direction, sequencing, and module routing. Issues
|
||||||
|
are the active backlog; this document is durable planning context and should be
|
||||||
|
mirrored to the Gitea wiki.
|
||||||
|
|
||||||
|
## Product Thesis
|
||||||
|
|
||||||
|
GovOPlaN should become a configurable operations platform for public
|
||||||
|
institutions. It should help an institution model real administrative
|
||||||
|
procedures, connect existing systems, keep durable evidence, and explain the
|
||||||
|
configured system to users.
|
||||||
|
|
||||||
|
The goal is not to replace every existing system. GovOPlaN should connect
|
||||||
|
existing systems, provide better workflows where the current landscape is weak,
|
||||||
|
and make administrative processes configurable, auditable, and reusable.
|
||||||
|
|
||||||
|
The product should first provide a reliable administrative spine:
|
||||||
|
|
||||||
|
- identities, roles, tenants, policy, audit, and governance
|
||||||
|
- forms, files, cases, workflow, tasks, templates, and records
|
||||||
|
- postboxes, notifications, mail, portal, appointments, and booking
|
||||||
|
- identity trust, role-bound postboxes, and eventually encrypted administrative
|
||||||
|
communication
|
||||||
|
- configuration packages that assemble modules into repeatable procedures
|
||||||
|
- docs that explain the configured system, not the full theoretical product
|
||||||
|
|
||||||
|
Domain modules should come after the spine can run a reference procedure end to
|
||||||
|
end.
|
||||||
|
|
||||||
|
The platform should be a governance-capable runtime for modules, connectors,
|
||||||
|
configuration, and administrative decisions. The kernel must stay free of domain
|
||||||
|
semantics while still providing the contracts needed for modules to explain what
|
||||||
|
they do, what they require, which effects they create, and how operators can
|
||||||
|
verify or reverse those effects.
|
||||||
|
|
||||||
|
## Design Principles
|
||||||
|
|
||||||
|
- Modules must stay independently installable, enableable, and disableable.
|
||||||
|
- Cross-module behavior should use core-mediated capabilities, commands,
|
||||||
|
events, DTOs, and UI contribution points rather than direct imports.
|
||||||
|
- Configuration packages should turn installed modules into concrete, reusable
|
||||||
|
administrative processes.
|
||||||
|
- Operators should be able to configure the platform through the UI.
|
||||||
|
- Every powerful configuration path needs preflight, preview, audit, rollback,
|
||||||
|
RBAC, and policy checks.
|
||||||
|
- Context, decision, consequence, and traceability must be visible together for
|
||||||
|
consequential actions. The full doctrine lives in
|
||||||
|
`INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md`.
|
||||||
|
- Automation must use governed action/effect contracts, not hidden side
|
||||||
|
effects. The first automation layer is defined in
|
||||||
|
`ACTION_EFFECT_AUTOMATION_LAYER.md` and should start in `govoplan-workflow`
|
||||||
|
unless a separate automation module becomes justified.
|
||||||
|
- Encrypted postboxes are a strategic target. Early postbox, access, and
|
||||||
|
identity-trust contracts should stay compatible with the E2EE architecture in
|
||||||
|
`POSTBOX_E2EE_ARCHITECTURE.md`.
|
||||||
|
- Integration should be a first-class product path: connect to existing
|
||||||
|
systems, consume their data, and publish governed outputs back to them.
|
||||||
|
- GovOPlaN should scale from a small local installation to a larger deployment
|
||||||
|
with separately scalable web, API, worker, storage, and database components.
|
||||||
|
|
||||||
|
## User Experience Direction
|
||||||
|
|
||||||
|
GovOPlaN should expose the full power of the platform without forcing
|
||||||
|
non-technical users to face every field, flag, and internal representation at
|
||||||
|
once. The default experience should feel guided, explainable, and calm. Expert
|
||||||
|
depth should remain available, but it should be layered behind deliberate
|
||||||
|
interaction patterns.
|
||||||
|
|
||||||
|
Core UX rules:
|
||||||
|
|
||||||
|
- Use progressive disclosure. Common decisions stay visible; advanced,
|
||||||
|
hazardous, or rarely used options live in collapsed panels, secondary steps,
|
||||||
|
or explicit advanced areas.
|
||||||
|
- Do not use raw JSON as the primary configuration UI. Every configurable value
|
||||||
|
should have an appropriate control, validation, and plain-language help.
|
||||||
|
Import/export and diagnostics may show JSON as a secondary artifact.
|
||||||
|
- Prefer guided flows over option dumps. Connector setup, package import,
|
||||||
|
module installation, policy changes, and destructive operations should use
|
||||||
|
wizards that explain what is happening, why it matters, and what will happen
|
||||||
|
next.
|
||||||
|
- Discover values when the system can infer them. For example, a Nextcloud file
|
||||||
|
connection should start with the base URL, discover the WebDAV endpoint, and
|
||||||
|
fill technical fields for review instead of asking the user to know them
|
||||||
|
upfront.
|
||||||
|
- Make explanations always available without making every screen verbose.
|
||||||
|
Inline helper text should be short; richer explanations should be reachable
|
||||||
|
through expandable help, tooltips, side panels, or review steps.
|
||||||
|
- Explain blocked actions in actionable language. A disabled control or failed
|
||||||
|
step should say what is missing, who can fix it, and where to go, for example
|
||||||
|
"A system administrator must allow this provider" or "Configure the provider
|
||||||
|
in Settings > File Providers before linking a folder here."
|
||||||
|
- Reuse visual language and placements consistently. Similar configuration,
|
||||||
|
policy, connection, credential, review, and confirmation flows should share
|
||||||
|
components, button placement, modal behavior, problem lists, and empty/error
|
||||||
|
states.
|
||||||
|
- Use modals and step flows for focused creation/editing where they reduce page
|
||||||
|
clutter. Reserve large always-open pages for overview, comparison, and
|
||||||
|
repeated administration work.
|
||||||
|
- Treat diagnostics as product UX. Validation results, preflight blockers,
|
||||||
|
policy explanations, permission denials, and missing capabilities should be
|
||||||
|
understandable to a non-technical operator before exposing internal details.
|
||||||
|
|
||||||
|
This is a product quality gate. New admin/configuration surfaces should not be
|
||||||
|
considered complete if they expose all options at once, require JSON editing,
|
||||||
|
hide why an action is unavailable, or use a one-off layout where a shared
|
||||||
|
pattern exists.
|
||||||
|
|
||||||
|
## Focus Rules
|
||||||
|
|
||||||
|
1. Build one reference journey per wave.
|
||||||
|
2. Do not implement a module because the repository exists.
|
||||||
|
3. Do not add module-to-module imports for optional behavior.
|
||||||
|
4. Every new domain module must justify its own semantics beyond `cases`,
|
||||||
|
`workflow`, `tasks`, `forms`, and `files`.
|
||||||
|
5. Prefer connector first when an external specialist system is likely to remain
|
||||||
|
the system of record.
|
||||||
|
6. Keep active work in Gitea issues; keep durable context in docs and synced
|
||||||
|
wiki pages.
|
||||||
|
7. A module moves from scaffold to implementation only when it has an owner,
|
||||||
|
reference journey, boundary notes, capability contracts, and testable MVP.
|
||||||
|
|
||||||
|
## Capability Map
|
||||||
|
|
||||||
|
| Capability | Likely owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Public application entry point | `govoplan-portal` |
|
||||||
|
| Structured forms and validation | `govoplan-forms` |
|
||||||
|
| Uploaded files and managed storage | `govoplan-files` |
|
||||||
|
| Case record and lifecycle | `govoplan-cases` |
|
||||||
|
| Workflow transitions and automation | `govoplan-workflow` |
|
||||||
|
| Action/effect catalogue and automation runner | first `govoplan-workflow`; possible future `govoplan-automation` if it outgrows workflow |
|
||||||
|
| Internal work queues and tasks | `govoplan-tasks` |
|
||||||
|
| Appointment proposals and booking | `govoplan-appointments`, `govoplan-calendar` |
|
||||||
|
| Postbox, email, and notifications | `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications` |
|
||||||
|
| Identity trust, device keys, and encrypted postbox key contracts | `govoplan-identity-trust`, `govoplan-access`, `govoplan-postbox` |
|
||||||
|
| Service directory/catalog | `govoplan-portal` |
|
||||||
|
| Permit/document generation | `govoplan-templates`, `govoplan-dms` |
|
||||||
|
| Payment capture and accounting handoff | `govoplan-payments`, `govoplan-ledger` |
|
||||||
|
| Roles, permissions, tenants, policy, audit | `govoplan-access`, `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` |
|
||||||
|
| External software integration | `govoplan-connectors` |
|
||||||
|
| Recurring extraction and transformation | possible future `govoplan-datasources`, possible future `govoplan-dataflow` |
|
||||||
|
| Reports, BI, and management visibility | `govoplan-reporting` |
|
||||||
|
|
||||||
|
## Configuration And Safety Target
|
||||||
|
|
||||||
|
The long-term target is that operators configure the platform through the UI
|
||||||
|
instead of editing files for normal operation.
|
||||||
|
|
||||||
|
UI-managed configuration should include:
|
||||||
|
|
||||||
|
- module installation, enablement, lifecycle state, and health
|
||||||
|
- tenants, users, groups, roles, policies, and permissions
|
||||||
|
- connectors, credentials, secret references, and external service tests
|
||||||
|
- workflows, forms, templates, task queues, schedules, and notifications
|
||||||
|
- configuration package import/export and environment-specific data collection
|
||||||
|
- retention, audit, privacy, maintenance mode, and safety controls
|
||||||
|
- deployment-visible settings such as public URLs, mail senders, storage
|
||||||
|
profiles, queues, and worker capabilities
|
||||||
|
|
||||||
|
Safety controls should include dry-run plans, field-level validation, policy
|
||||||
|
explanations, two-person approval for destructive changes, versioned
|
||||||
|
configuration history, rollback paths, audit events, and maintenance-mode
|
||||||
|
guards.
|
||||||
|
|
||||||
|
The initial safety metadata contract lives in
|
||||||
|
`govoplan_core.core.configuration_safety`. It classifies known configuration
|
||||||
|
fields as UI-managed or deployment-managed, assigns risk levels, marks secret
|
||||||
|
handling as reference-only or env-only, and declares dry-run, policy
|
||||||
|
explanation, audit, approval, rollback-history, maintenance-mode, and RBAC
|
||||||
|
requirements. Admin UI editors should consume this metadata before exposing
|
||||||
|
powerful settings.
|
||||||
|
|
||||||
|
The initial executable guardrail path is `plan_configuration_change(...)`,
|
||||||
|
exposed through:
|
||||||
|
|
||||||
|
- `GET /api/v1/admin/configuration-safety`
|
||||||
|
- `POST /api/v1/admin/configuration-safety/plan`
|
||||||
|
|
||||||
|
The planner reports missing scopes, dry-run requirements, maintenance-mode
|
||||||
|
requirements, two-person approval status, secret-reference violations,
|
||||||
|
rollback-history requirements, policy explanations, and audit event names before
|
||||||
|
an editor applies a high-impact configuration change.
|
||||||
|
|
||||||
|
## Reference Journeys
|
||||||
|
|
||||||
|
The roadmap should be driven by three journeys.
|
||||||
|
|
||||||
|
### Journey 1: Permit To Payment
|
||||||
|
|
||||||
|
This is the primary public-administration journey.
|
||||||
|
|
||||||
|
1. A person applies for a permit through the public portal.
|
||||||
|
2. The applicant uploads required files and submits structured form data.
|
||||||
|
3. Submission creates a case, a workflow instance, and an internal task.
|
||||||
|
4. Completing the task creates a postbox message, a notification, and an email
|
||||||
|
notification with an appointment proposal.
|
||||||
|
5. The applicant accepts an appointment, which updates the calendar and the
|
||||||
|
workflow state.
|
||||||
|
6. During the appointment, the case is opened and the permit is generated from
|
||||||
|
a governed template.
|
||||||
|
7. The payment is processed and linked to the case and accounting handoff.
|
||||||
|
8. The permit, payment evidence, communication history, audit trail, retention
|
||||||
|
state, and records evidence remain available according to policy.
|
||||||
|
|
||||||
|
This journey proves the platform can coordinate modules without core knowing
|
||||||
|
module internals.
|
||||||
|
|
||||||
|
### Journey 2: Training To Certificate
|
||||||
|
|
||||||
|
This is the best university-administration and internal-administration journey.
|
||||||
|
|
||||||
|
1. Course or training offer is planned.
|
||||||
|
2. Room, trainer, resource, and capacity are booked.
|
||||||
|
3. Participants register or are assigned.
|
||||||
|
4. Attendance is tracked.
|
||||||
|
5. Certificate or participation confirmation is issued.
|
||||||
|
6. Evidence remains available through records, files, audit, and docs.
|
||||||
|
|
||||||
|
This journey keeps `booking`, `resources`, `learning`, and `certificates`
|
||||||
|
focused instead of becoming broad ERP replacements.
|
||||||
|
|
||||||
|
### Journey 3: Report To Resolution
|
||||||
|
|
||||||
|
This is the internal operations and municipal issue-reporting journey.
|
||||||
|
|
||||||
|
1. A person reports an issue.
|
||||||
|
2. The issue is triaged into helpdesk, facilities, assets, or a case.
|
||||||
|
3. Work is assigned, tracked, and escalated.
|
||||||
|
4. Evidence, communication, and status updates are preserved.
|
||||||
|
5. Reports show workload, SLA, recurring problems, and completion.
|
||||||
|
|
||||||
|
This journey prevents `helpdesk`, `issue-reporting`, `facilities`, and `assets`
|
||||||
|
from becoming disconnected ticket silos.
|
||||||
|
|
||||||
|
## Roadmap Waves
|
||||||
|
|
||||||
|
### Wave 0: Platform Spine
|
||||||
|
|
||||||
|
Goal: make the platform safe to configure and extend.
|
||||||
|
|
||||||
|
Refine:
|
||||||
|
|
||||||
|
- `govoplan-core`: module discovery, capabilities, events, migrations, release
|
||||||
|
catalog, configuration package runtime, WebUI shell.
|
||||||
|
- `govoplan-access`: identities, sessions, API keys, users, groups, roles,
|
||||||
|
memberships, function assignments, delegation, RBAC decisions.
|
||||||
|
- `govoplan-tenancy`: tenant and organizational-unit boundaries.
|
||||||
|
- `govoplan-identity-trust`: initial trust contracts for device keys, public key
|
||||||
|
directory, assurance, and later encrypted postbox key access.
|
||||||
|
- `govoplan-policy`: policy sources, policy decisions, retention inputs.
|
||||||
|
- `govoplan-audit`: audit sink, audit routes, trace context, retention
|
||||||
|
cooperation.
|
||||||
|
- `govoplan-admin`: UI-managed configuration with preflight, rollback, audit,
|
||||||
|
and approval controls.
|
||||||
|
- `govoplan-dashboard`: configurable user home assembled from module-provided
|
||||||
|
widgets, with core providing only a minimal fallback when the module is absent.
|
||||||
|
- `govoplan-docs`: configured-system documentation and evidence-aware help.
|
||||||
|
- `govoplan-ops`: deployment profiles, health checks, worker split, sizing
|
||||||
|
assumptions.
|
||||||
|
|
||||||
|
Do not expand domain scope in this wave. The output is a dependable platform
|
||||||
|
surface for later modules.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- module enablement and capability lookup are stable
|
||||||
|
- configuration package preflight works for at least one simple package
|
||||||
|
- audit and policy decisions are visible in admin flows
|
||||||
|
- access distinguishes identity, account, function, role, and right in durable
|
||||||
|
contracts
|
||||||
|
- action/effect automation contracts are specified before hidden side effects
|
||||||
|
spread across modules
|
||||||
|
- docs can show installed/enabled modules and configured routes
|
||||||
|
|
||||||
|
### Wave 1: Permit-To-Payment MVP
|
||||||
|
|
||||||
|
Goal: one complete public-administration process.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-forms-runtime`: form submissions, drafts, validation, attachments,
|
||||||
|
and submission evidence.
|
||||||
|
2. `govoplan-portal`: public authenticated and unauthenticated entry points.
|
||||||
|
3. `govoplan-files`: upload, evidence links, file spaces, and permissioned
|
||||||
|
access.
|
||||||
|
4. `govoplan-cases`: case record, status, assignments, deadlines, and case
|
||||||
|
evidence.
|
||||||
|
5. `govoplan-workflow`: state machine, transitions, commands, and module
|
||||||
|
handoff.
|
||||||
|
6. `govoplan-tasks`: work queues, assignments, due dates, and follow-ups.
|
||||||
|
7. `govoplan-templates`: permit/decision document generation.
|
||||||
|
8. `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications`: applicant and
|
||||||
|
internal communication, with the postbox model compatible with later E2EE
|
||||||
|
and role-bound access.
|
||||||
|
9. `govoplan-calendar`, `govoplan-appointments`, `govoplan-booking`: appointment
|
||||||
|
or booking handoff for the reference process.
|
||||||
|
10. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment
|
||||||
|
capture, payment evidence, and accounting/e-invoice handoff.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- a permit-to-payment package can be installed in a local demo
|
||||||
|
- every step has audit evidence
|
||||||
|
- every module integration uses capabilities, events, or DTOs
|
||||||
|
- user-facing documentation describes only the configured process
|
||||||
|
|
||||||
|
### Wave 2: Booking And Resource Operations
|
||||||
|
|
||||||
|
Goal: make time, capacity, and resource allocation a reusable product area.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-booking`: booking rules, capacity, quotas, waitlists, cancellation,
|
||||||
|
attendance, no-shows, and approvals.
|
||||||
|
2. `govoplan-resources`: rooms, equipment, vehicles, counters, labs, capacity,
|
||||||
|
availability, and maintenance blocks.
|
||||||
|
3. `govoplan-calendar`: event and availability integration.
|
||||||
|
4. `govoplan-appointments`: appointment-specific booking surfaces.
|
||||||
|
5. `govoplan-facilities`: buildings, rooms, maintenance, access zones,
|
||||||
|
inspections, and defects.
|
||||||
|
6. `govoplan-assets`: inventory, assignment, lifecycle, maintenance, and handover
|
||||||
|
evidence.
|
||||||
|
|
||||||
|
Reference journey: reserve a room/resource for a training or appointment and
|
||||||
|
preserve all booking evidence.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- bookable resources are not hardcoded into appointments
|
||||||
|
- booking decisions are explainable and auditable
|
||||||
|
- resources can be blocked by maintenance or facility status
|
||||||
|
|
||||||
|
### Wave 3: Training And Certificates
|
||||||
|
|
||||||
|
Goal: support university-style and internal public-sector training without
|
||||||
|
building a full learning-management system first.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-learning`: course planning, course booking, participant lists,
|
||||||
|
trainers, attendance, evaluations, and learning records.
|
||||||
|
2. `govoplan-certificates`: participation confirmations, certificates,
|
||||||
|
verification, revocation, and evidence links.
|
||||||
|
3. `govoplan-booking`, `govoplan-resources`, `govoplan-calendar`: reuse booking
|
||||||
|
and room/resource allocation.
|
||||||
|
4. `govoplan-templates`, `govoplan-files`, `govoplan-records`: certificate
|
||||||
|
generation and durable retention.
|
||||||
|
|
||||||
|
Reference journey: plan a course, book resources, register participants, track
|
||||||
|
attendance, and issue a certificate.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- attendance can produce certificate eligibility
|
||||||
|
- certificates can be verified later
|
||||||
|
- course operations do not depend on university-specific assumptions
|
||||||
|
|
||||||
|
### Wave 4: Report-To-Resolution Operations
|
||||||
|
|
||||||
|
Goal: cover internal support and public issue reporting.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-issue-reporting`: public/internal reports, categories, intake,
|
||||||
|
location, evidence, and triage.
|
||||||
|
2. `govoplan-helpdesk`: service desk tickets, queues, SLAs, assignments,
|
||||||
|
escalation, and resolution evidence.
|
||||||
|
3. `govoplan-facilities` and `govoplan-assets`: issue handoff to maintenance or
|
||||||
|
asset lifecycle.
|
||||||
|
4. `govoplan-cases`, `govoplan-workflow`, `govoplan-tasks`: escalation into
|
||||||
|
formal administrative matters.
|
||||||
|
5. `govoplan-reporting`: workload, SLA, recurring defects, and status reports.
|
||||||
|
|
||||||
|
Reference journey: report a facility issue, triage it, assign work, resolve it,
|
||||||
|
and report recurring defects.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- issue reporting is not just a generic form inbox
|
||||||
|
- helpdesk tickets can remain lightweight unless formal case handling is needed
|
||||||
|
- operational metrics exist without custom SQL
|
||||||
|
|
||||||
|
### Wave 5: Records, DMS, Transparency
|
||||||
|
|
||||||
|
Goal: make evidence legally and organizationally durable.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-dms`: document lifecycle, collaboration, versions, locks, approvals,
|
||||||
|
and DMS connectors.
|
||||||
|
2. `govoplan-records`: file plans, classification, retention schedules, disposal
|
||||||
|
holds, archive handoff, and records evidence.
|
||||||
|
3. `govoplan-policy` and `govoplan-audit`: retention policy, legal hold, audit
|
||||||
|
traceability.
|
||||||
|
4. `govoplan-search`: permissioned cross-module discovery.
|
||||||
|
5. `govoplan-transparency`: FOI/public-records requests, redaction, publication,
|
||||||
|
and disclosure evidence.
|
||||||
|
|
||||||
|
Reference journey: close a case, classify records, apply retention, and answer a
|
||||||
|
transparency request.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- files, documents, and records are not treated as the same concept
|
||||||
|
- transparency disclosure can redact and explain evidence
|
||||||
|
- retention and archive handoff are policy-governed
|
||||||
|
|
||||||
|
### Wave 6: Finance, Procurement, Contracts, Grants
|
||||||
|
|
||||||
|
Goal: cover the back-office procedures around spending, obligations, funding,
|
||||||
|
and revenue without replacing a finance system too early.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-procurement`: purchase requests, approvals, vendor comparison,
|
||||||
|
tender references, goods receipt, and handoff.
|
||||||
|
2. `govoplan-contracts`: contract register, obligations, renewals, reminders,
|
||||||
|
and responsible units.
|
||||||
|
3. `govoplan-grants`: funding programs, applications, eligibility, awards,
|
||||||
|
milestones, claims, and reporting duties.
|
||||||
|
4. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment,
|
||||||
|
accounting, and e-invoice handoff.
|
||||||
|
5. `govoplan-erp`: integration with the actual system of record, not a full ERP
|
||||||
|
replacement unless later justified.
|
||||||
|
|
||||||
|
Reference journey: purchase or grant approval through obligation tracking and
|
||||||
|
financial handoff.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- ERP remains an integration point unless GovOPlaN must own semantics
|
||||||
|
- contracts and grants produce obligations, deadlines, and reporting tasks
|
||||||
|
- financial evidence links back to cases, files, and audit
|
||||||
|
|
||||||
|
### Wave 7: Institutional Governance And Participation
|
||||||
|
|
||||||
|
Goal: support public bodies, municipalities, ministries, and universities in
|
||||||
|
formal coordination.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-committee`: committees, boards, councils, agendas, minutes,
|
||||||
|
decisions, voting, and follow-up tasks.
|
||||||
|
2. `govoplan-consultation`: hearings, public consultations, stakeholder
|
||||||
|
feedback, formal comments, response matrices, and publication workflows.
|
||||||
|
3. `govoplan-campaign`: communication campaigns, recipients, templates, review,
|
||||||
|
sending control, and reports.
|
||||||
|
4. `govoplan-addresses`: persons, organizations, contacts, distribution lists,
|
||||||
|
and recipient import/export.
|
||||||
|
5. `govoplan-portal` and `govoplan-transparency`: public participation and
|
||||||
|
publication surfaces.
|
||||||
|
|
||||||
|
Reference journey: prepare a committee decision, publish consultation material,
|
||||||
|
collect feedback, document the decision, and assign follow-up tasks.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- decisions create durable tasks and records
|
||||||
|
- consultations can publish evidence without exposing restricted material
|
||||||
|
- campaign and address behavior stays optional and capability-based
|
||||||
|
|
||||||
|
### Wave 8: Integration, Data, And Reporting
|
||||||
|
|
||||||
|
Goal: connect the platform to real institutional landscapes.
|
||||||
|
|
||||||
|
Refine:
|
||||||
|
|
||||||
|
- `govoplan-connectors`: integration catalog, connector metadata, adapter
|
||||||
|
lifecycle, test harnesses.
|
||||||
|
- `govoplan-idm`, `govoplan-identity-trust`: identity provider, directory, and
|
||||||
|
assurance integration.
|
||||||
|
- `govoplan-fit-connect`, `govoplan-xoev`, `govoplan-xta-osci`: public-sector
|
||||||
|
transport and standards integration.
|
||||||
|
- `govoplan-reporting`: operational reporting, scheduled outputs, exports, and
|
||||||
|
dashboard data.
|
||||||
|
- `govoplan-search`: permissioned cross-module discovery.
|
||||||
|
|
||||||
|
Create only when justified:
|
||||||
|
|
||||||
|
- `govoplan-datasources`: source catalog, connection profiles, schema discovery,
|
||||||
|
freshness, provenance.
|
||||||
|
- `govoplan-dataflow`: transformations, validation, lineage, scheduled runs,
|
||||||
|
publication outputs.
|
||||||
|
- `govoplan-projects`: only if OpenProject connectors and existing cases/tasks
|
||||||
|
cannot cover the required semantics.
|
||||||
|
|
||||||
|
Reference journey: monthly data extraction, transformation, validation, approval,
|
||||||
|
publication, and reporting.
|
||||||
|
|
||||||
|
Recurring extraction/transformation should start as a configuration package
|
||||||
|
across connectors, files, workflow, reporting, and templates. The package should
|
||||||
|
register sources, declare schemas, define mapping/validation versions, schedule
|
||||||
|
runs, produce previewable diffs, write governed outputs, and preserve lineage,
|
||||||
|
hashes, operator actions, and audit evidence. Create `govoplan-datasources` or
|
||||||
|
`govoplan-dataflow` only after this work exposes repeated contracts that do not
|
||||||
|
belong to existing modules.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- connector catalog exists before building many adapters
|
||||||
|
- dataflow is created only after recurring transformation becomes product
|
||||||
|
behavior
|
||||||
|
- reporting consumes governed sources with provenance
|
||||||
|
|
||||||
|
## Implementation Gates
|
||||||
|
|
||||||
|
Before a module receives implementation work, it needs:
|
||||||
|
|
||||||
|
- one reference journey step it owns
|
||||||
|
- ownership and boundary notes
|
||||||
|
- initial manifest and permission list
|
||||||
|
- capability contracts or API DTOs
|
||||||
|
- audit and policy expectations
|
||||||
|
- Gitea issues for MVP tasks
|
||||||
|
- docs page that can sync to the wiki
|
||||||
|
- focused tests that can run from the core environment
|
||||||
|
|
||||||
|
Before a module becomes release-included, it needs:
|
||||||
|
|
||||||
|
- installable Python package metadata where applicable
|
||||||
|
- WebUI package metadata where applicable
|
||||||
|
- module manifest entry point
|
||||||
|
- release catalog entry
|
||||||
|
- smoke test or permutation test
|
||||||
|
- no required imports from optional modules
|
||||||
|
|
||||||
|
## Priority Order Summary
|
||||||
|
|
||||||
|
1. Stabilize the platform spine.
|
||||||
|
2. Deliver permit-to-payment MVP.
|
||||||
|
3. Build booking and resource operations.
|
||||||
|
4. Add learning and certificates.
|
||||||
|
5. Add issue reporting and helpdesk.
|
||||||
|
6. Add records, DMS, search, and transparency.
|
||||||
|
7. Add procurement, contracts, grants, and finance handoff.
|
||||||
|
8. Add committee and consultation workflows.
|
||||||
|
9. Expand integration, dataflow, reporting, and operations.
|
||||||
|
|
||||||
|
## Deliberate Deferrals
|
||||||
|
|
||||||
|
Defer these until a reference journey proves the need:
|
||||||
|
|
||||||
|
- full ERP replacement
|
||||||
|
- native project management beyond connector support
|
||||||
|
- broad BI/dataflow platform
|
||||||
|
- every possible public-sector protocol adapter
|
||||||
|
- rich LMS behavior beyond training administration
|
||||||
|
- full qualified digital signing/trust services beyond the identity-trust and
|
||||||
|
encrypted-postbox contracts needed for early architecture safety
|
||||||
|
- mobile apps
|
||||||
|
- AI assistants embedded into workflows
|
||||||
|
|
||||||
|
These may become important, but they should not distract from the first complete
|
||||||
|
administrative journeys.
|
||||||
|
|
||||||
|
## Module And Integration Routing
|
||||||
|
|
||||||
|
This table maps current module and integration ideas to existing GovOPlaN
|
||||||
|
repositories or to explicit missing-module decisions.
|
||||||
|
|
||||||
|
| Idea | Owner | Tracking |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Government operations backbone reference model | `govoplan-core` | `add-ideas/govoplan-core#213` |
|
||||||
|
| Permit-to-payment configuration package | `govoplan-core` plus participating modules | `add-ideas/govoplan-core#214` |
|
||||||
|
| Fully UI-managed configuration with safety controls | `govoplan-admin`, `govoplan-core`, `govoplan-policy`, `govoplan-access`, `govoplan-audit` | `add-ideas/govoplan-core#218` |
|
||||||
|
| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` |
|
||||||
|
| Interface ethics and decision-consequence doctrine | `govoplan-core` plus all UI-owning modules | `add-ideas/govoplan-core#227` |
|
||||||
|
| Action/effect automation layer | first `govoplan-workflow`; possible future `govoplan-automation` | `add-ideas/govoplan-workflow#1` |
|
||||||
|
| E2EE role/function postbox architecture | `govoplan-postbox`, `govoplan-identity-trust`, `govoplan-access`, `govoplan-policy`, `govoplan-audit` | `add-ideas/govoplan-postbox#15`, `add-ideas/govoplan-identity-trust#1` |
|
||||||
|
| Identity, account, function, role, right semantic model | `govoplan-access` | `add-ideas/govoplan-access#9` |
|
||||||
|
| Role-based service directory/catalog | `govoplan-portal` | `add-ideas/govoplan-portal#1` |
|
||||||
|
| Unified inbox across tasks, postbox, notifications, and portal | `govoplan-core` coordination plus owning modules | `add-ideas/govoplan-tasks#1`, `add-ideas/govoplan-notifications#1` |
|
||||||
|
| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` |
|
||||||
|
| Native project-management module decision | connector-first through `govoplan-connectors`; no native project module yet | `add-ideas/govoplan-core#196`, `add-ideas/govoplan-connectors#1` |
|
||||||
|
| Datasources for databases, CSV, files, APIs | no repository yet; start with connectors/files/reporting and create `govoplan-datasources` only after the first package proves shared source-catalog ownership | `add-ideas/govoplan-core#197` |
|
||||||
|
| Dataflow for pipelines, BI, publication | no repository yet; start with workflow/reporting/connectors and create `govoplan-dataflow` only after repeated pipeline/lineage contracts emerge | `add-ideas/govoplan-core#198` |
|
||||||
|
| Monthly datasource and transformation workflows | first as configuration package across connectors, files, workflow, reporting, and templates | `add-ideas/govoplan-core#216` |
|
||||||
|
| Templates for letters, emails, forms, reports | `govoplan-templates`, separate from reporting | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-templates#1` |
|
||||||
|
| Reporting and BI | `govoplan-reporting`, separate from templates | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-reporting#1` |
|
||||||
|
| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` |
|
||||||
|
| Public-sector software integration catalogue | `govoplan-connectors` with core strategy index | `add-ideas/govoplan-core#191`, `add-ideas/govoplan-connectors#2` |
|
||||||
|
| Public-sector integration landscape catalogue | `govoplan-connectors` with core tracking | `add-ideas/govoplan-core#215` |
|
||||||
|
| Cases module concept | `govoplan-cases` | `add-ideas/govoplan-core#174` |
|
||||||
|
| Workflow module concept | `govoplan-workflow` | `add-ideas/govoplan-core#175` |
|
||||||
|
| Connectors module concept | `govoplan-connectors` | `add-ideas/govoplan-core#176` |
|
||||||
|
| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` |
|
||||||
|
| Consume sources and become a governed source | `govoplan-connectors` plus possible future `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` |
|
||||||
|
| Governed connector configuration, dry-run, and simulation runtime | `govoplan-connectors` | `add-ideas/govoplan-connectors#6` |
|
||||||
|
| Terminfindung and meeting scheduling polls | `govoplan-scheduling`; calendar primitives remain in calendar | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-scheduling#1` |
|
||||||
|
| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-calendar#1` |
|
||||||
|
| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-appointments#1` |
|
||||||
|
| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` |
|
||||||
|
| Forms | `govoplan-forms` for definitions and `govoplan-forms-runtime` for submissions/runtime behavior | `add-ideas/govoplan-core#194`, `add-ideas/govoplan-forms#1` |
|
||||||
|
| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` |
|
||||||
|
| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` |
|
||||||
|
| OpenDesk stack integration map | integration profile across IDM/access, mail/calendar, files/DMS, and connectors; not a monolithic module | `add-ideas/govoplan-core#195`, `add-ideas/govoplan-connectors#5` |
|
||||||
|
| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` |
|
||||||
|
| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` |
|
||||||
|
| Scalability profiles and autoscaling readiness | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#217` |
|
||||||
|
| Hardware sizing matrix and requirements calculator | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#219` |
|
||||||
|
| Collaboration suite integration strategy | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow`, `govoplan-tasks`, `govoplan-appointments`, `govoplan-calendar` | `add-ideas/govoplan-core#220` |
|
||||||
|
| Install/runtime configuration contract | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#19` |
|
||||||
|
| Installer/deployment operator flow | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#26` |
|
||||||
|
| Production-like deployment documentation | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#28` |
|
||||||
|
|
||||||
|
Boundary rationale lives in `MODULE_ARCHITECTURE.md`. Current decisions:
|
||||||
|
|
||||||
|
- templates and reporting are separate modules
|
||||||
|
- RSS/source consume-publish starts in connectors; datasources/dataflow are not
|
||||||
|
repositories yet
|
||||||
|
- calendar, scheduling, and appointments are three separate modules
|
||||||
|
- forms definitions and forms runtime are separate responsibilities
|
||||||
|
- OpenDesk is an integration profile across modules, not a monolithic module
|
||||||
|
- OpenProject is connector-first; no native projects module yet
|
||||||
|
- public-sector integration strategy stays in core; executable catalogue work
|
||||||
|
lives in connectors
|
||||||
|
- encrypted postbox and identity-trust are strategic contracts, not mail-module
|
||||||
|
behavior
|
||||||
|
- automation starts as workflow-owned action/effect execution and may split into
|
||||||
|
a dedicated module only after the runner becomes broader than workflow
|
||||||
|
|
||||||
|
The following modules are intentionally not created yet:
|
||||||
|
|
||||||
|
- `govoplan-datasources`
|
||||||
|
- `govoplan-dataflow`
|
||||||
|
- `govoplan-projects`
|
||||||
|
|
||||||
|
Create a repository only after a concrete implementation package proves that
|
||||||
|
existing connector, files, reporting, workflow, or task ownership is too narrow.
|
||||||
|
|
||||||
|
Core keeps the strategy index in
|
||||||
|
`PUBLIC_SECTOR_INTEGRATION_STRATEGY.md`: integration postures, default
|
||||||
|
ownership, and prioritization rules. `govoplan-connectors` owns the detailed
|
||||||
|
target inventory and connector entry shape in
|
||||||
|
`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`.
|
||||||
|
|
||||||
|
Release composition and tag-only repository handling are documented in
|
||||||
|
`RELEASE_DEPENDENCIES.md`.
|
||||||
|
|
||||||
|
## Next Practical Work
|
||||||
|
|
||||||
|
The next planning step should create or update Gitea issues for Wave 0 and Wave
|
||||||
|
1 only. Later waves should stay as roadmap context until the permit-to-payment
|
||||||
|
MVP is demonstrable.
|
||||||
|
|
||||||
|
Recommended immediate issue buckets:
|
||||||
|
|
||||||
|
- platform spine hardening
|
||||||
|
- configuration package preflight and rollback
|
||||||
|
- forms-runtime MVP
|
||||||
|
- portal submission MVP
|
||||||
|
- cases/workflow/tasks integration MVP
|
||||||
|
- template-generated decision document
|
||||||
|
- postbox/notification handoff
|
||||||
|
- appointment/booking handoff
|
||||||
|
- payment evidence handoff
|
||||||
|
- configured documentation for the reference process
|
||||||
102
docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md
Normal file
102
docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
# GovOPlaN Interface Ethics And Design Doctrine
|
||||||
|
|
||||||
|
This document captures the product-level design doctrine for GovOPlaN. It is
|
||||||
|
more durable than an individual screen design and should guide admin,
|
||||||
|
configuration, workflow, policy, portal, and operational UI decisions.
|
||||||
|
|
||||||
|
GovOPlaN is meant to support administrative responsibility. The interface must
|
||||||
|
therefore make context, decision, consequence, and traceability visible enough
|
||||||
|
that users can act deliberately instead of being pushed through opaque
|
||||||
|
automation.
|
||||||
|
|
||||||
|
## Core Doctrine
|
||||||
|
|
||||||
|
1. Context, decision, and consequence belong together.
|
||||||
|
2. Decisions should not silently happen.
|
||||||
|
3. Transparency comes before convenience when rights, duties, records, money,
|
||||||
|
access, or legal effects are involved.
|
||||||
|
4. Explicit state is preferable to implicit state.
|
||||||
|
5. Navigation is not consent.
|
||||||
|
6. Responsibility cannot be delegated to the system.
|
||||||
|
7. Traceability is part of the action, not a later reporting feature.
|
||||||
|
8. Context loss is a product defect.
|
||||||
|
|
||||||
|
These rules do not mean every screen should become verbose. They mean the
|
||||||
|
interface must expose the right explanation at the moment of decision and keep
|
||||||
|
technical detail available without making it the default surface.
|
||||||
|
|
||||||
|
## Decision Surface Contract
|
||||||
|
|
||||||
|
Any action that changes records, rights, policies, retention, communication,
|
||||||
|
payments, external systems, or workflow state should answer these questions
|
||||||
|
before execution:
|
||||||
|
|
||||||
|
- What object, person, organization, or process is affected?
|
||||||
|
- Which authority or role allows the actor to do this?
|
||||||
|
- What will change immediately?
|
||||||
|
- What downstream effects may happen?
|
||||||
|
- Can the action be undone, superseded, or only corrected later?
|
||||||
|
- What evidence or audit entry will be created?
|
||||||
|
- Which policy, configuration, or missing capability blocks the action?
|
||||||
|
- Who can resolve a blocker?
|
||||||
|
|
||||||
|
The answer may be shown through inline labels, a review step, a side panel, a
|
||||||
|
problem list, or a confirmation dialog. The important point is that consequence
|
||||||
|
and responsibility are not hidden behind a generic submit button.
|
||||||
|
|
||||||
|
## Contestability
|
||||||
|
|
||||||
|
Administrative decisions are often contestable or reviewable. GovOPlaN should
|
||||||
|
therefore preserve the path from input to decision:
|
||||||
|
|
||||||
|
- source data and attachments
|
||||||
|
- workflow state and task assignment
|
||||||
|
- policy decisions and source path
|
||||||
|
- actor and delegation context
|
||||||
|
- generated document/template version
|
||||||
|
- external handoff result
|
||||||
|
- notification or postbox delivery evidence
|
||||||
|
- retention and record classification state
|
||||||
|
|
||||||
|
Where a user sees a decision, they should be able to reach the provenance that
|
||||||
|
explains how the system got there. This is especially important for denials,
|
||||||
|
locks, calculated defaults, generated documents, payment state, retention
|
||||||
|
state, and access decisions.
|
||||||
|
|
||||||
|
## Anti-Patterns
|
||||||
|
|
||||||
|
Avoid these patterns in GovOPlaN interfaces:
|
||||||
|
|
||||||
|
- Magical buttons that execute multiple side effects without preview.
|
||||||
|
- Process tunnels that hide where the user is in an administrative procedure.
|
||||||
|
- Silent automation that changes external systems without an audit-visible
|
||||||
|
command record.
|
||||||
|
- Friendly hiding that removes complexity at the cost of obscuring authority,
|
||||||
|
consequence, or accountability.
|
||||||
|
- Disabled controls without actionable explanation.
|
||||||
|
- Configuration screens that ask users to edit raw JSON as the normal path.
|
||||||
|
|
||||||
|
## Automation Rule
|
||||||
|
|
||||||
|
Automation must use the same governed action surface as a human actor. The
|
||||||
|
system may execute actions as a system actor, but it must still run through
|
||||||
|
policy checks, capability contracts, audit, idempotency, and failure handling.
|
||||||
|
|
||||||
|
When an automated decision is not clear, GovOPlaN should create a manual
|
||||||
|
exception, task, or review item instead of guessing silently.
|
||||||
|
|
||||||
|
## Relationship To UI Components
|
||||||
|
|
||||||
|
Shared components should make this doctrine easy to follow:
|
||||||
|
|
||||||
|
- preflight and problem-list components for blockers
|
||||||
|
- policy source path and effective decision displays
|
||||||
|
- action review panels for consequence preview
|
||||||
|
- audit/provenance links on decision outputs
|
||||||
|
- guided dialogs for risky configuration
|
||||||
|
- disabled-action explanations with actor and next step
|
||||||
|
- confirmation dialogs that distinguish reversible, corrective, and destructive
|
||||||
|
actions
|
||||||
|
|
||||||
|
The UI/UX decision ledger defines concrete implementation rules. This doctrine
|
||||||
|
defines why those rules exist.
|
||||||
File diff suppressed because it is too large
Load Diff
115
docs/POLICY_CONTRACTS.md
Normal file
115
docs/POLICY_CONTRACTS.md
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
# GovOPlaN Policy Contracts
|
||||||
|
|
||||||
|
GovOPlaN has several policy families that are moving out of core into owning
|
||||||
|
modules. The shared kernel contract keeps their decision and provenance shape
|
||||||
|
consistent while each module still owns its domain rules.
|
||||||
|
|
||||||
|
## Current Policy Inventory
|
||||||
|
|
||||||
|
| Policy area | Current owner | Runtime surface | Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Privacy retention | `govoplan-policy` implementation and routes, with compatibility helpers in core | `/api/v1/admin/privacy-retention/policies/{scope}` and `/explain`; capability `policy.privacyRetention` | System, tenant, user, group, and campaign sources merge into the effective retention policy. Parent locks block lower-level widening. |
|
||||||
|
| Mail profile policy | `govoplan-mail` | `/api/v1/mail/policies/{scope}` | Uses the same source-step path format for system, tenant, owner, and campaign provenance. |
|
||||||
|
| RBAC/access policy | `govoplan-access` | access capabilities in `govoplan_core.core.access` | Permission decisions should use access capability contracts. Explain responses should adopt `PolicyDecision` when an API-level explanation is added. |
|
||||||
|
| Governance defaults | `govoplan-admin` plus `govoplan-access` materializer | admin settings, governance template routes, access materialization capability | System governance can block tenant-local groups, roles, and API keys. |
|
||||||
|
| Delegation and ownership policy | access/campaign/mail/files modules | capability checks and owner-scoped APIs | Source provenance should use this contract when policies become externally explainable. |
|
||||||
|
|
||||||
|
## Policy Decision
|
||||||
|
|
||||||
|
The shared DTO lives in `govoplan_core.core.policy.PolicyDecision`.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"allowed": false,
|
||||||
|
"reason": "Parent retention policy locks lower-level changes.",
|
||||||
|
"source_path": [
|
||||||
|
{
|
||||||
|
"scope_type": "system",
|
||||||
|
"scope_id": null,
|
||||||
|
"path": "system",
|
||||||
|
"label": "System",
|
||||||
|
"applied_fields": ["allow_lower_level_limits"],
|
||||||
|
"policy": {}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"requirements": ["raw_campaign_json_retention_days"],
|
||||||
|
"details": {
|
||||||
|
"blocked_fields": ["raw_campaign_json_retention_days"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`allowed` is the effective answer for the checked action. `reason` is a stable,
|
||||||
|
human-readable summary. `source_path` lists the policy sources that explain the
|
||||||
|
answer. `requirements` lists machine-readable blockers or prerequisites, and
|
||||||
|
`details` carries domain-specific structured context.
|
||||||
|
|
||||||
|
Every source step should be concrete enough for an operator to understand the
|
||||||
|
decision without knowing internal merge rules. Use real scope labels such as
|
||||||
|
`System`, `Tenant`, `Owner user`, `Group`, or a campaign/profile name. Include
|
||||||
|
the stable `path`, the fields applied by that step, and the local policy
|
||||||
|
fragment that caused them. This lets UIs render explanations like
|
||||||
|
`System: Allow > Tenant: Deny without override` without additional lookups.
|
||||||
|
If a policy family cannot expose the full local fragment for security reasons,
|
||||||
|
it must still include a redacted structured value that identifies the applied
|
||||||
|
field and the effective allow/deny or lock state.
|
||||||
|
|
||||||
|
## Source Path Format
|
||||||
|
|
||||||
|
Policy source paths are stable string identifiers for provenance steps:
|
||||||
|
|
||||||
|
- `system`
|
||||||
|
- `<scope_type>:<url-encoded-scope-id>`
|
||||||
|
|
||||||
|
Supported scope types are `system`, `tenant`, `user`, `group`, and `campaign`.
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- `tenant:4a45b4fe-1d86-43ce-9d10-6022333f4d4b`
|
||||||
|
- `campaign:campaign%2Fwith%20space`
|
||||||
|
|
||||||
|
Use `policy_source_path()` and `parse_policy_source_path()` instead of building
|
||||||
|
or splitting these strings manually.
|
||||||
|
|
||||||
|
## Retention Explain Endpoint
|
||||||
|
|
||||||
|
`GET /api/v1/admin/privacy-retention/policies/{scope_type}/explain` returns:
|
||||||
|
|
||||||
|
- `scope_type` and optional `scope_id`
|
||||||
|
- `decision`, using the shared `PolicyDecision` shape
|
||||||
|
- `effective_policy`
|
||||||
|
- optional `parent_policy`
|
||||||
|
- `effective_policy_sources`
|
||||||
|
- `parent_policy_sources`
|
||||||
|
- `blocked_fields`
|
||||||
|
|
||||||
|
The endpoint is read-only. Enforcement remains in the existing policy write
|
||||||
|
path. For lower-level scopes, `blocked_fields` is derived from the parent
|
||||||
|
policy's `allow_lower_level_limits`; clients can use it to disable local
|
||||||
|
controls before attempting a write.
|
||||||
|
|
||||||
|
The retention implementation lives in `govoplan-policy`
|
||||||
|
(`govoplan_policy.backend.retention`). Core keeps
|
||||||
|
`govoplan_core.privacy.retention` only as a compatibility facade for older
|
||||||
|
imports. Effective/scoped retention behavior dispatches through the
|
||||||
|
`policy.privacyRetention` capability; core does not import policy implementation
|
||||||
|
code as a hidden fallback when the module is disabled or no runtime is active.
|
||||||
|
New backend code should import policy-owned retention behavior from
|
||||||
|
`govoplan-policy` or request the capability, not add new implementation logic
|
||||||
|
to core.
|
||||||
|
|
||||||
|
## Frontend Contract
|
||||||
|
|
||||||
|
Policy UIs must:
|
||||||
|
|
||||||
|
- render effective source provenance when `effective_policy_sources` is present
|
||||||
|
- display a field-level path when the source data is shown next to a specific
|
||||||
|
setting, using concrete source labels and stop at the first non-overridable
|
||||||
|
deny/lock
|
||||||
|
- disable local field controls when the parent policy sets that field's
|
||||||
|
lower-level limit to `false`
|
||||||
|
- avoid sending locked fields or re-enable attempts in save payloads
|
||||||
|
- show inherited values separately from local overrides
|
||||||
|
|
||||||
|
The core WebUI helper `privacyRetentionParentAllowsField()` centralizes the
|
||||||
|
field-lock decision used by the retention editor and its lightweight module
|
||||||
|
tests.
|
||||||
134
docs/POSTBOX_E2EE_ARCHITECTURE.md
Normal file
134
docs/POSTBOX_E2EE_ARCHITECTURE.md
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
# Postbox End-To-End Encryption Architecture
|
||||||
|
|
||||||
|
This document records the strategic encryption target for GovOPlaN postboxes.
|
||||||
|
It does not require the first postbox implementation to ship full E2EE, but it
|
||||||
|
defines the architecture so early data models and APIs do not make the stronger
|
||||||
|
model impossible.
|
||||||
|
|
||||||
|
The core principle is that a postbox can become a trusted administrative
|
||||||
|
communication channel without requiring the server to see plaintext content.
|
||||||
|
The server may route, store, authorize, audit, retain, and expire messages while
|
||||||
|
message bodies and attachments remain client-encrypted.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- asynchronous encrypted delivery for internal and portal-facing postboxes
|
||||||
|
- personal, organizational, role-bound, and function-bound postboxes
|
||||||
|
- attachments encrypted with the message
|
||||||
|
- access based on current role/function membership when configured
|
||||||
|
- honest retraction and expiry semantics
|
||||||
|
- auditable key access, delivery, and fetch events
|
||||||
|
- support for external recipients without platform accounts
|
||||||
|
- replaceable identity and trust providers
|
||||||
|
|
||||||
|
## Envelope Model
|
||||||
|
|
||||||
|
The target model is envelope encryption:
|
||||||
|
|
||||||
|
- Generate one random data encryption key per message or attachment set.
|
||||||
|
- Encrypt content with an authenticated encryption algorithm.
|
||||||
|
- Wrap the data encryption key for each authorized recipient or role mailbox.
|
||||||
|
- Store only ciphertext, wrapped keys, signed manifests, and governed metadata
|
||||||
|
on the server.
|
||||||
|
|
||||||
|
Algorithm choices should remain replaceable behind a crypto profile. The first
|
||||||
|
profile should prefer standard, reviewed primitives such as HPKE for key
|
||||||
|
wrapping and AEAD encryption for content.
|
||||||
|
|
||||||
|
## Identity And Device Keys
|
||||||
|
|
||||||
|
The platform should distinguish:
|
||||||
|
|
||||||
|
- account identity
|
||||||
|
- tenant membership
|
||||||
|
- role/function assignment
|
||||||
|
- device key
|
||||||
|
- postbox binding
|
||||||
|
|
||||||
|
Identity providers and directories can authenticate users and provide membership
|
||||||
|
facts, but they must not see postbox private keys or message plaintext.
|
||||||
|
|
||||||
|
The trust layer should provide:
|
||||||
|
|
||||||
|
- public key directory
|
||||||
|
- account or identity signing keys
|
||||||
|
- per-device encryption keys
|
||||||
|
- device registration and revocation
|
||||||
|
- key rotation and epoch tracking
|
||||||
|
- recovery policy hooks
|
||||||
|
|
||||||
|
## Role And Function Postboxes
|
||||||
|
|
||||||
|
Role-bound access needs special handling. A postbox can be bound to an
|
||||||
|
organizational unit and a role or function. Current members can access current
|
||||||
|
messages according to policy; former members should lose access to not-yet
|
||||||
|
fetched material when revocation is still technically enforceable.
|
||||||
|
|
||||||
|
The target design should support role encryption keys or an equivalent
|
||||||
|
rewrapping service:
|
||||||
|
|
||||||
|
- sender encrypts the content key for the role/function postbox
|
||||||
|
- access service verifies current membership and required assurance
|
||||||
|
- trust service rewraps the content key to the actor's current device key
|
||||||
|
- audit records the key access decision and fetch event
|
||||||
|
|
||||||
|
Key epochs are required when role membership changes. Older messages may remain
|
||||||
|
readable according to policy, but new access must use the current epoch.
|
||||||
|
|
||||||
|
## External Recipients
|
||||||
|
|
||||||
|
External recipients may need one-time or time-limited access without a full
|
||||||
|
platform account. The target model should support capability links or invitation
|
||||||
|
tokens that are:
|
||||||
|
|
||||||
|
- scoped to specific message or attachment resources
|
||||||
|
- time-limited
|
||||||
|
- optionally one-time
|
||||||
|
- protected by an out-of-band secret, passphrase, or stronger external identity
|
||||||
|
proof
|
||||||
|
- revocable before key fetch
|
||||||
|
- fully audited
|
||||||
|
|
||||||
|
## Retraction Semantics
|
||||||
|
|
||||||
|
GovOPlaN should be honest about retraction.
|
||||||
|
|
||||||
|
Before a recipient fetches a key or decrypts content, the system can revoke
|
||||||
|
tokens, remove wrapped-key access, expire links, and delete ciphertext according
|
||||||
|
to retention policy.
|
||||||
|
|
||||||
|
After a recipient has decrypted or copied plaintext, the system cannot make the
|
||||||
|
recipient forget it. The platform can only record access, revoke future access,
|
||||||
|
notify parties, and apply legal or organizational controls.
|
||||||
|
|
||||||
|
The UI must explain this distinction whenever it offers expiry, retraction, or
|
||||||
|
message withdrawal.
|
||||||
|
|
||||||
|
## Server Responsibilities
|
||||||
|
|
||||||
|
The server remains important even when content is encrypted:
|
||||||
|
|
||||||
|
- store ciphertext and signed manifests
|
||||||
|
- store routing and policy metadata
|
||||||
|
- enforce access before key release or rewrapping
|
||||||
|
- provide public key directory access
|
||||||
|
- emit notifications without plaintext content
|
||||||
|
- record audit events
|
||||||
|
- enforce retention and expiry where possible
|
||||||
|
- expose diagnostics for delivery and key-access failures
|
||||||
|
|
||||||
|
## Module Ownership
|
||||||
|
|
||||||
|
- `govoplan-postbox` owns postbox bindings, postbox messages, message metadata,
|
||||||
|
and postbox UI.
|
||||||
|
- `govoplan-identity-trust` owns device keys, public key directory, key epochs,
|
||||||
|
and assurance integration.
|
||||||
|
- `govoplan-access` owns current identity, membership, function, delegation, and
|
||||||
|
permission decisions.
|
||||||
|
- `govoplan-policy` owns retention, retraction, and security policy decisions.
|
||||||
|
- `govoplan-audit` owns durable audit traces.
|
||||||
|
- `govoplan-files` owns managed file storage when encrypted postbox attachments
|
||||||
|
are backed by file objects.
|
||||||
|
|
||||||
|
No module should import another module's internals to decrypt content. All
|
||||||
|
interaction must use capabilities, DTOs, and audited service contracts.
|
||||||
276
docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md
Normal file
276
docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
# Public-Sector Integration Strategy
|
||||||
|
|
||||||
|
GovOPlaN should integrate with the existing public-sector software landscape
|
||||||
|
before deciding to replace specialist workflows. This document is the core
|
||||||
|
strategy index. The executable connector catalogue lives in
|
||||||
|
`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`.
|
||||||
|
|
||||||
|
## Strategy Labels
|
||||||
|
|
||||||
|
Use one or more of these labels for every external system family:
|
||||||
|
|
||||||
|
- `integrate`: GovOPlaN talks to the system through a stable API/protocol.
|
||||||
|
- `link`: GovOPlaN stores external references and opens the external system for
|
||||||
|
source-of-truth work.
|
||||||
|
- `import`: GovOPlaN consumes data or files into governed module storage.
|
||||||
|
- `synchronize`: GovOPlaN keeps selected records aligned both ways or through a
|
||||||
|
source-of-truth rule.
|
||||||
|
- `replace selected workflow`: GovOPlaN may own a narrow workflow where the
|
||||||
|
external product is weak, but does not replace the whole product family.
|
||||||
|
- `no first-class support`: GovOPlaN only stores manual references unless a
|
||||||
|
deployment project creates a specific connector.
|
||||||
|
|
||||||
|
## Initial Classification
|
||||||
|
|
||||||
|
| System family | Examples | Default strategy | Likely owner |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| File providers | SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3 | integrate, import, link | `govoplan-files`, connector inventory in `govoplan-connectors` |
|
||||||
|
| Project/task management | OpenProject, Jira, Redmine, Microsoft Planner | link, synchronize selected records, replace selected workflow only after proof | `govoplan-connectors`, later `govoplan-tasks` or workflow modules |
|
||||||
|
| Identity providers | LDAP, Active Directory, OIDC, SAML, OpenDesk IDM | integrate, synchronize | `govoplan-idm`, `govoplan-access` |
|
||||||
|
| Mail and groupware | IMAP/SMTP, Open-Xchange, Exchange/M365, CalDAV/CardDAV | integrate, link | `govoplan-mail`, `govoplan-calendar`, `govoplan-connectors` |
|
||||||
|
| DMS/e-file/archive | d.velop/d.3, enaio, ELO, Fabasoft, CMIS, VIS/eAkte | link, import, synchronize selected metadata | `govoplan-dms`, `govoplan-files`, `govoplan-records` |
|
||||||
|
| ERP/finance/procurement | SAP, MACH, Infoma, DATEV, procurement feeds | export, import, synchronize; do not replace by default | `govoplan-erp`, `govoplan-procurement`, `govoplan-ledger`, `govoplan-payments` |
|
||||||
|
| Public-sector transport | FIT-Connect, XTA/OSCI, Peppol access points | integrate, publish, receive | dedicated protocol modules plus `govoplan-connectors` inventory |
|
||||||
|
| Standards registries | XRepository, XOE/V catalogues | link, import metadata/cache | `govoplan-connectors`, `govoplan-xoev` |
|
||||||
|
| Publication/data exchange | RSS, open-data APIs, API feeds, CSV/Excel drops | consume, publish, transform | proposed `govoplan-datasources`, proposed `govoplan-dataflow`, `govoplan-reporting` |
|
||||||
|
| Collaboration suites | Matrix, Jitsi, BigBlueButton, Nextcloud Talk, Collabora/OnlyOffice | integrate, link; native behavior only for governed evidence | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow` |
|
||||||
|
| Specialist Fachverfahren | register-specific and domain-specific systems | link first; integrate/import when a real project supplies contracts | domain module or deployment-specific connector |
|
||||||
|
|
||||||
|
## Landscape Catalogue
|
||||||
|
|
||||||
|
This catalogue is intentionally implementation-oriented. Each entry records the
|
||||||
|
first API/auth/data assumptions needed to turn an inventory entry into a
|
||||||
|
connector or module issue.
|
||||||
|
|
||||||
|
### Citizen And Service Portals
|
||||||
|
|
||||||
|
- Strategy: integrate/link first; replace selected intake workflow only when a
|
||||||
|
GovOPlaN portal package owns the complete journey.
|
||||||
|
- Protocol/API surface: REST/JSON APIs, form submission webhooks, OIDC/SAML
|
||||||
|
login, eID interfaces where available, file-upload callbacks, case-status
|
||||||
|
callbacks.
|
||||||
|
- Auth model: OIDC/SAML service clients, signed webhook secrets, tenant-scoped
|
||||||
|
API keys, later eID/trust-provider handoff.
|
||||||
|
- Data shape: applicant identity reference, application form payload,
|
||||||
|
attachment references, consent declarations, status events, receipt IDs.
|
||||||
|
- Deployment assumptions: externally reachable HTTPS, reverse proxy, portal DMZ
|
||||||
|
separation, strict CSRF/origin settings, large upload path.
|
||||||
|
- Risks: personal data exposure, duplicate identity mapping, partial
|
||||||
|
submissions, upload malware, inconsistent portal status models.
|
||||||
|
- MVP test path: submit a test application with one file, create a form
|
||||||
|
submission/case/task stub, return a receipt and status reference.
|
||||||
|
- Owner/priority: `govoplan-portal`, `govoplan-forms-runtime`,
|
||||||
|
`govoplan-files`, Wave 1.
|
||||||
|
|
||||||
|
### DMS, E-File, Records, And Archives
|
||||||
|
|
||||||
|
- Strategy: link/import/synchronize selected metadata; do not replace the DMS by
|
||||||
|
default.
|
||||||
|
- Protocol/API surface: CMIS, WebDAV, vendor REST APIs, S3/object archive
|
||||||
|
staging, file-plan export/import, archive handoff APIs.
|
||||||
|
- Auth model: service accounts, OAuth/OIDC where supported, mTLS for regulated
|
||||||
|
archives, secret references for vendor tokens.
|
||||||
|
- Data shape: document ID, version, file-plan/classification code, retention
|
||||||
|
metadata, owner/case reference, external URL, checksum, lock/legal-hold state.
|
||||||
|
- Deployment assumptions: usually internal network or VPN, strict storage
|
||||||
|
quotas, existing retention policies, archive immutability requirements.
|
||||||
|
- Risks: record duplication, broken legal hold, permission mismatch, version
|
||||||
|
drift, destructive retention/export mistakes.
|
||||||
|
- MVP test path: create a connector inventory entry, test read-only metadata
|
||||||
|
lookup, link one GovOPlaN file/case evidence item to an external document.
|
||||||
|
- Owner/priority: `govoplan-dms`, `govoplan-records`, `govoplan-files`,
|
||||||
|
`govoplan-connectors`, Wave 2/5.
|
||||||
|
|
||||||
|
### ERP, Finance, Procurement, And Accounting
|
||||||
|
|
||||||
|
- Strategy: export/import/synchronize selected records; replacement only by
|
||||||
|
narrow domain decision.
|
||||||
|
- Protocol/API surface: vendor REST/SOAP APIs, CSV/XML batch exchange, SFTP,
|
||||||
|
XRechnung/Peppol, XBestellung/procurement feeds, payment reconciliation files.
|
||||||
|
- Auth model: service accounts, client certificates, mTLS, SFTP keys, token
|
||||||
|
references, environment-specific account separation.
|
||||||
|
- Data shape: debtor/creditor reference, payment request, invoice, order,
|
||||||
|
budget/cost-center code, booking status, receipt/evidence reference.
|
||||||
|
- Deployment assumptions: batch windows, finance-system approval workflows,
|
||||||
|
test tenants often separated from production by vendor process.
|
||||||
|
- Risks: financial posting errors, double export, tax/legal data retention,
|
||||||
|
inconsistent master data, irreversible accounting handoff.
|
||||||
|
- MVP test path: dry-run export of one payment/accounting handoff file with
|
||||||
|
checksum, validation report, and no remote posting.
|
||||||
|
- Owner/priority: `govoplan-payments`, `govoplan-ledger`,
|
||||||
|
`govoplan-xrechnung`, `govoplan-erp`, `govoplan-procurement`, Wave 1/6.
|
||||||
|
|
||||||
|
### Identity, IAM, And Directory Services
|
||||||
|
|
||||||
|
- Strategy: integrate/synchronize; access remains GovOPlaN's local
|
||||||
|
authorization boundary.
|
||||||
|
- Protocol/API surface: LDAP, Active Directory, SCIM, OIDC, SAML, OpenDesk IDM
|
||||||
|
APIs, group membership sync, account deactivation feeds.
|
||||||
|
- Auth model: bind accounts, service clients, OIDC/SAML metadata, SCIM tokens,
|
||||||
|
certificate-backed clients where required.
|
||||||
|
- Data shape: account, user, group, membership, role claim, tenant/org-unit
|
||||||
|
mapping, status, external directory ID.
|
||||||
|
- Deployment assumptions: directory is usually internal; identity provider may
|
||||||
|
be organization-wide and not GovOPlaN-owned.
|
||||||
|
- Risks: privilege escalation through group mapping, stale memberships, account
|
||||||
|
collision, deprovisioning latency, tenant-boundary mistakes.
|
||||||
|
- MVP test path: read-only directory profile test, map one external group to a
|
||||||
|
tenant group, show a dry-run membership diff.
|
||||||
|
- Owner/priority: `govoplan-idm`, `govoplan-access`, Wave 1.
|
||||||
|
|
||||||
|
### Groupware, Mail, Calendar, And Collaboration
|
||||||
|
|
||||||
|
- Strategy: integrate/link; native behavior only where GovOPlaN needs governed
|
||||||
|
evidence or process state.
|
||||||
|
- Protocol/API surface: IMAP/SMTP, CalDAV/CardDAV, Open-Xchange APIs, Microsoft
|
||||||
|
Graph/EWS, Matrix APIs, Jitsi/BigBlueButton APIs, Collabora/OnlyOffice
|
||||||
|
integration points.
|
||||||
|
- Auth model: service accounts, delegated OAuth/OIDC, app passwords, mailbox
|
||||||
|
credentials, groupware-specific tokens, secret references.
|
||||||
|
- Data shape: mailbox folder/message references, event/free-busy data, meeting
|
||||||
|
URL, chat room ID, participant list, document-editing session reference.
|
||||||
|
- Deployment assumptions: often internal/existing tenant infrastructure; mail
|
||||||
|
and calendar may be separate from identity even in OpenDesk-style stacks.
|
||||||
|
- Risks: mail credential exposure, calendar privacy, double invitations, room
|
||||||
|
booking conflicts, chat/document data escaping retention rules.
|
||||||
|
- MVP test path: profile test for mailbox/calendar reachability, read-only
|
||||||
|
folder/free-busy lookup, create a non-production event/message draft.
|
||||||
|
- Owner/priority: `govoplan-mail`, `govoplan-calendar`,
|
||||||
|
`govoplan-connectors`, Wave 1/2.
|
||||||
|
|
||||||
|
### Payment And Public Cashier Systems
|
||||||
|
|
||||||
|
- Strategy: integrate/export/import; keep the payment provider or cashier as
|
||||||
|
source of settlement truth.
|
||||||
|
- Protocol/API surface: payment provider APIs, redirect/callback flows,
|
||||||
|
reconciliation files, SEPA/export formats, cash-register/cashier interfaces.
|
||||||
|
- Auth model: provider API keys, signed webhooks, client certificates, mTLS,
|
||||||
|
tenant-specific merchant accounts.
|
||||||
|
- Data shape: payment intent, amount/currency, payer reference, provider
|
||||||
|
transaction ID, settlement status, receipt, refund/cancellation reference.
|
||||||
|
- Deployment assumptions: public callback URLs, strict environment separation,
|
||||||
|
PCI-sensitive providers, finance reconciliation cadence.
|
||||||
|
- Risks: duplicate charges, callback replay, amount mismatch, refund workflow
|
||||||
|
gaps, evidence-retention mistakes.
|
||||||
|
- MVP test path: sandbox payment intent, signed callback verification, receipt
|
||||||
|
evidence link, reconciliation dry-run.
|
||||||
|
- Owner/priority: `govoplan-payments`, `govoplan-ledger`, Wave 1/6.
|
||||||
|
|
||||||
|
### Reporting, BI, Open Data, And Publication
|
||||||
|
|
||||||
|
- Strategy: consume/publish/transform; native reporting owns GovOPlaN views, not
|
||||||
|
every external BI product.
|
||||||
|
- Protocol/API surface: SQL read replicas, CSV/Excel export/import, REST APIs,
|
||||||
|
RSS/Atom, open-data APIs, SFTP/WebDAV publication targets.
|
||||||
|
- Auth model: read-only DB users, API tokens, SFTP keys, OAuth clients, public
|
||||||
|
anonymous publication profiles where appropriate.
|
||||||
|
- Data shape: dataset metadata, schema/version, report parameters, generated
|
||||||
|
file references, publication URL, freshness/lineage, validation results.
|
||||||
|
- Deployment assumptions: publication can be public or internal; generated
|
||||||
|
datasets need retention and provenance.
|
||||||
|
- Risks: leaking restricted data, stale publications, schema drift, expensive
|
||||||
|
queries, untraceable manual transformations.
|
||||||
|
- MVP test path: publish one report/export as a governed file plus RSS/Atom
|
||||||
|
entry with checksum, timestamp, and permission check.
|
||||||
|
- Owner/priority: `govoplan-reporting`, `govoplan-connectors`, possible future
|
||||||
|
`govoplan-datasources`/`govoplan-dataflow`, Wave 2.
|
||||||
|
|
||||||
|
### Public-Sector Protocols And Registries
|
||||||
|
|
||||||
|
- Strategy: integrate/publish/receive; protocol modules own protocol semantics.
|
||||||
|
- Protocol/API surface: FIT-Connect, XTA/OSCI, XRepository, XOE/V, XRechnung,
|
||||||
|
XBestellung, Peppol, registry-specific Fachverfahren APIs.
|
||||||
|
- Auth model: certificates, mTLS, service accounts, destination credentials,
|
||||||
|
protocol-specific trust anchors and key rotation.
|
||||||
|
- Data shape: transport envelope, payload schema/version, destination IDs,
|
||||||
|
receipt/acknowledgement, message status, standard-specific metadata.
|
||||||
|
- Deployment assumptions: regulated trust chains, test/prod endpoint
|
||||||
|
separation, formal onboarding, strict logging and retention expectations.
|
||||||
|
- Risks: invalid schemas, failed delivery receipts, certificate expiry, wrong
|
||||||
|
destination routing, protocol version drift.
|
||||||
|
- MVP test path: validate a sample payload against a schema, test endpoint
|
||||||
|
reachability in sandbox, store receipt/evidence reference.
|
||||||
|
- Owner/priority: `govoplan-fit-connect`, `govoplan-xoev`,
|
||||||
|
`govoplan-xrechnung`, `govoplan-xta-osci`, `govoplan-connectors`, Wave 1/2.
|
||||||
|
|
||||||
|
### File Providers And Shared Storage
|
||||||
|
|
||||||
|
- Strategy: integrate/import/link; files module owns GovOPlaN file semantics.
|
||||||
|
- Protocol/API surface: SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3,
|
||||||
|
local/object storage profiles.
|
||||||
|
- Auth model: service accounts, user credentials, app tokens, OAuth where
|
||||||
|
supported, secret references, environment variables for deployment-managed
|
||||||
|
credentials.
|
||||||
|
- Data shape: file ID/path, provider object ID, checksum, MIME type, size,
|
||||||
|
version/ETag, owner, permission snapshot, imported file reference.
|
||||||
|
- Deployment assumptions: internal networks, large files, existing shares,
|
||||||
|
variable permissions, provider-specific rate limits.
|
||||||
|
- Risks: permission mismatch, stale imports, overwrites, duplicate files, path
|
||||||
|
traversal, storage growth.
|
||||||
|
- MVP test path: profile test, list folder, import one file into governed
|
||||||
|
storage, keep provider reference and checksum.
|
||||||
|
- Owner/priority: `govoplan-files`, `govoplan-connectors`, Wave 0/1.
|
||||||
|
|
||||||
|
### Project, Task, And Case-Adjacent Systems
|
||||||
|
|
||||||
|
- Strategy: connector-first for OpenProject/Jira/Redmine; native module only
|
||||||
|
when GovOPlaN owns project semantics.
|
||||||
|
- Protocol/API surface: OpenProject API v3, webhooks, Jira/Redmine REST APIs,
|
||||||
|
Microsoft Graph for Planner/Project where applicable.
|
||||||
|
- Auth model: API tokens, OAuth/OIDC apps, webhook secrets, service accounts.
|
||||||
|
- Data shape: project ID, work package/task ID, status, assignee reference,
|
||||||
|
external URL, version/lock token, publish/sync trace.
|
||||||
|
- Deployment assumptions: external project tool remains source of truth for
|
||||||
|
broad project management; GovOPlaN links selected records.
|
||||||
|
- Risks: task duplication, bidirectional sync conflicts, permission mismatch,
|
||||||
|
over-mirroring comments/attachments.
|
||||||
|
- MVP test path: OpenProject profile test, project/work-package lookup,
|
||||||
|
external-reference round-trip.
|
||||||
|
- Owner/priority: `govoplan-connectors`, later `govoplan-tasks`/workflow/cases
|
||||||
|
consumers, Wave 0/2.
|
||||||
|
|
||||||
|
### Specialist Fachverfahren
|
||||||
|
|
||||||
|
- Strategy: link first; integrate/import only when a deployment project supplies
|
||||||
|
concrete contracts and a domain owner.
|
||||||
|
- Protocol/API surface: vendor APIs, CSV/XML batch imports, SFTP, database
|
||||||
|
views, message queues, protocol-specific transports.
|
||||||
|
- Auth model: usually service accounts, VPN, mTLS, SFTP keys, or vendor tokens.
|
||||||
|
- Data shape: domain-specific record IDs, status, applicant/person references,
|
||||||
|
file/evidence references, case/status events.
|
||||||
|
- Deployment assumptions: strongly local/vendor-specific, often no stable test
|
||||||
|
API, data model differs by jurisdiction.
|
||||||
|
- Risks: brittle vendor contracts, legal source-of-truth ambiguity, high
|
||||||
|
customization cost, migration expectations.
|
||||||
|
- MVP test path: inventory entry and manual external-reference link; require a
|
||||||
|
project-specific connector issue before automation.
|
||||||
|
- Owner/priority: domain module or deployment-specific connector, case by case.
|
||||||
|
|
||||||
|
## Prioritization Rules
|
||||||
|
|
||||||
|
1. Start with connectors that unblock Wave 0 or Wave 1 reference journeys.
|
||||||
|
2. Prefer open standards and self-hosted/open-source APIs where they are common
|
||||||
|
in public-sector deployments.
|
||||||
|
3. Treat inventory-only entries as useful because operators need a map of their
|
||||||
|
software landscape even before automation exists.
|
||||||
|
4. Keep connector code in the owning connector/protocol module. Domain modules
|
||||||
|
consume capabilities, DTOs, external references, and events through core.
|
||||||
|
5. Every executable connector needs health diagnostics, secret-reference
|
||||||
|
handling, lifecycle state, audit events, and retirement behavior.
|
||||||
|
|
||||||
|
## Connector Catalogue Handoff
|
||||||
|
|
||||||
|
`govoplan-connectors` owns the detailed catalogue entry shape:
|
||||||
|
|
||||||
|
- connector type key
|
||||||
|
- category and owner module
|
||||||
|
- supported directions and trigger modes
|
||||||
|
- credential and secret handling
|
||||||
|
- health check and diagnostics payload
|
||||||
|
- external-reference shape
|
||||||
|
- required capabilities and optional module combinations
|
||||||
|
- lifecycle support
|
||||||
|
|
||||||
|
Core should only keep strategy, routing, and cross-module architecture notes.
|
||||||
|
Connector implementation and public-sector target inventory belong in
|
||||||
|
`govoplan-connectors`.
|
||||||
@@ -1,291 +0,0 @@
|
|||||||
# Multi Seal Mail - Current RBAC and Resource-Access Model
|
|
||||||
|
|
||||||
**Updated:** 2026-06-16
|
|
||||||
**Current migration head:** `f5a6b7c8d9e0`
|
|
||||||
|
|
||||||
## Authorization Equation
|
|
||||||
|
|
||||||
An operation is permitted only when every applicable layer allows it:
|
|
||||||
|
|
||||||
```text
|
|
||||||
effective role/API-key capability
|
|
||||||
AND resource ownership/share access
|
|
||||||
AND workflow state
|
|
||||||
AND active governance/policy constraints
|
|
||||||
```
|
|
||||||
|
|
||||||
RBAC answers what an actor may do. ACLs answer which resource the actor may do it to. Workflow state and policy decide whether the operation is currently valid.
|
|
||||||
|
|
||||||
## Identity and Scope
|
|
||||||
|
|
||||||
```text
|
|
||||||
Account global login identity
|
|
||||||
+- User membership tenant-local identity
|
|
||||||
+- direct tenant roles
|
|
||||||
+- active group memberships
|
|
||||||
| +- inherited tenant roles
|
|
||||||
+- tenant-local API keys
|
|
||||||
|
|
||||||
Account
|
|
||||||
+- direct system-role assignments
|
|
||||||
```
|
|
||||||
|
|
||||||
A browser session has one active tenant membership. System privileges do not silently grant tenant data access. API keys remain tenant-local and receive the intersection of their configured scopes and their owner's live tenant scopes on every request.
|
|
||||||
|
|
||||||
## Wildcards
|
|
||||||
|
|
||||||
```text
|
|
||||||
tenant:* every canonical tenant permission
|
|
||||||
system:* every canonical system permission
|
|
||||||
* legacy alias interpreted as tenant:* only
|
|
||||||
```
|
|
||||||
|
|
||||||
Tenant wildcards never grant system permissions.
|
|
||||||
|
|
||||||
## Canonical Tenant Permissions - 53
|
|
||||||
|
|
||||||
### Campaigns
|
|
||||||
|
|
||||||
```text
|
|
||||||
campaign:read
|
|
||||||
campaign:create
|
|
||||||
campaign:update
|
|
||||||
campaign:copy
|
|
||||||
campaign:archive
|
|
||||||
campaign:delete
|
|
||||||
campaign:share
|
|
||||||
campaign:validate
|
|
||||||
campaign:build
|
|
||||||
campaign:review
|
|
||||||
campaign:send_test
|
|
||||||
campaign:queue
|
|
||||||
campaign:control
|
|
||||||
campaign:send
|
|
||||||
campaign:retry
|
|
||||||
campaign:reconcile
|
|
||||||
```
|
|
||||||
|
|
||||||
### Recipients
|
|
||||||
|
|
||||||
```text
|
|
||||||
recipients:read
|
|
||||||
recipients:write
|
|
||||||
recipients:import
|
|
||||||
recipients:export
|
|
||||||
```
|
|
||||||
|
|
||||||
### Files
|
|
||||||
|
|
||||||
```text
|
|
||||||
files:read
|
|
||||||
files:download
|
|
||||||
files:upload
|
|
||||||
files:organize
|
|
||||||
files:share
|
|
||||||
files:delete
|
|
||||||
files:admin
|
|
||||||
```
|
|
||||||
|
|
||||||
### Reports and Audit
|
|
||||||
|
|
||||||
```text
|
|
||||||
reports:read
|
|
||||||
reports:export
|
|
||||||
reports:send
|
|
||||||
audit:read
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mail Servers
|
|
||||||
|
|
||||||
```text
|
|
||||||
mail_servers:read
|
|
||||||
mail_servers:use
|
|
||||||
mail_servers:test
|
|
||||||
mail_servers:write
|
|
||||||
mail_servers:manage_credentials
|
|
||||||
```
|
|
||||||
|
|
||||||
### Tenant Administration
|
|
||||||
|
|
||||||
```text
|
|
||||||
admin:users:read
|
|
||||||
admin:users:create
|
|
||||||
admin:users:update
|
|
||||||
admin:users:suspend
|
|
||||||
|
|
||||||
admin:groups:read
|
|
||||||
admin:groups:write
|
|
||||||
admin:groups:manage_members
|
|
||||||
|
|
||||||
admin:roles:read
|
|
||||||
admin:roles:write
|
|
||||||
admin:roles:assign
|
|
||||||
|
|
||||||
admin:api_keys:read
|
|
||||||
admin:api_keys:create
|
|
||||||
admin:api_keys:revoke
|
|
||||||
|
|
||||||
admin:settings:read
|
|
||||||
admin:settings:write
|
|
||||||
admin:policies:read
|
|
||||||
admin:policies:write
|
|
||||||
```
|
|
||||||
|
|
||||||
## Canonical System Permissions - 18
|
|
||||||
|
|
||||||
```text
|
|
||||||
system:tenants:read
|
|
||||||
system:tenants:create
|
|
||||||
system:tenants:update
|
|
||||||
system:tenants:suspend
|
|
||||||
|
|
||||||
system:accounts:read
|
|
||||||
system:accounts:create
|
|
||||||
system:accounts:update
|
|
||||||
system:accounts:suspend
|
|
||||||
|
|
||||||
system:roles:read
|
|
||||||
system:roles:write
|
|
||||||
system:roles:assign
|
|
||||||
|
|
||||||
system:access:read
|
|
||||||
system:access:assign
|
|
||||||
|
|
||||||
system:audit:read
|
|
||||||
system:settings:read
|
|
||||||
system:settings:write
|
|
||||||
system:governance:read
|
|
||||||
system:governance:write
|
|
||||||
```
|
|
||||||
|
|
||||||
`system:access:*` remains as a compatibility/read and assignment boundary for cross-tenant/system access handling. It is not a separate primary UI area.
|
|
||||||
|
|
||||||
## Default Tenant Roles
|
|
||||||
|
|
||||||
- **Owner:** `tenant:*`. At least one active operational owner must remain.
|
|
||||||
- **Tenant administrator:** settings, policies, users, groups, roles and API keys plus read access to campaigns/files/reports/audit. Real delivery remains separately delegable.
|
|
||||||
- **Administrator (legacy):** all tenant permissions for upgraded installations.
|
|
||||||
- **Access administrator:** membership and assignment management within delegation limits.
|
|
||||||
- **Campaign manager:** prepare, validate and build campaigns; no review approval or real delivery by default.
|
|
||||||
- **Reviewer:** inspect and approve prepared campaign messages.
|
|
||||||
- **Sender:** mock-test, queue, control, send, retry and reconcile prepared campaigns; can use/test approved mail profiles.
|
|
||||||
- **File manager:** managed file operations without campaign delivery rights.
|
|
||||||
- **Viewer:** read campaigns, recipients, files and reports.
|
|
||||||
- **Auditor:** read campaigns, recipient evidence, reports and audit records; export detailed evidence.
|
|
||||||
|
|
||||||
## Default System Roles
|
|
||||||
|
|
||||||
- **System owner:** `system:*`, protected. At least one active account must retain it.
|
|
||||||
- **System administrator:** all specific system permissions, editable and not protected.
|
|
||||||
- **System auditor:** read-only system registry/settings/governance/audit role, editable.
|
|
||||||
|
|
||||||
## Delegation Ceiling
|
|
||||||
|
|
||||||
For role definition, assignment and API-key creation:
|
|
||||||
|
|
||||||
```text
|
|
||||||
requested scopes subset of actor delegateable scopes
|
|
||||||
```
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
1. Tenant roles may contain tenant scopes only.
|
|
||||||
2. System roles may contain system scopes only.
|
|
||||||
3. Definition rights and assignment rights are separate.
|
|
||||||
4. Group definition and group membership management are separate.
|
|
||||||
5. API-key scopes are intersected with the owner's current effective scopes on every request.
|
|
||||||
6. Suspended accounts, users, tenants or groups stop contributing access immediately.
|
|
||||||
7. Administrative updates are field-sensitive; a user with only status authority cannot change role assignments.
|
|
||||||
|
|
||||||
## Campaign Ownership and ACLs
|
|
||||||
|
|
||||||
A campaign has exactly one owner:
|
|
||||||
|
|
||||||
```text
|
|
||||||
owner user OR owner group
|
|
||||||
```
|
|
||||||
|
|
||||||
Additional active shares may target users or groups with `read` or `write`.
|
|
||||||
|
|
||||||
Resolution:
|
|
||||||
|
|
||||||
- owner user: read and write;
|
|
||||||
- member of owner group: read and write;
|
|
||||||
- explicit read share: read;
|
|
||||||
- explicit write share: read and write;
|
|
||||||
- `tenant:*`: tenant-wide ACL bypass;
|
|
||||||
- ordinary campaign permission without ownership/share: no object access.
|
|
||||||
|
|
||||||
ACLs do not add capabilities. A write share still needs the specific permission for update, validation, review, send, report, retry or reconciliation.
|
|
||||||
|
|
||||||
## Sensitive Recipient Boundary
|
|
||||||
|
|
||||||
Recipient-complete campaign JSON, message data and job detail require `recipients:read`. Recipient edits require `recipients:write`; exports require `recipients:export`; import is reserved for the dedicated recipient import/list workflow.
|
|
||||||
|
|
||||||
## Files
|
|
||||||
|
|
||||||
| Permission | Operations |
|
|
||||||
|---|---|
|
|
||||||
| `files:read` | list, search, inspect, resolve metadata |
|
|
||||||
| `files:download` | download file bytes and generated ZIP archives |
|
|
||||||
| `files:upload` | upload files and ZIP contents |
|
|
||||||
| `files:organize` | create folders, rename, move, copy and bulk rename |
|
|
||||||
| `files:share` | create/revoke file shares |
|
|
||||||
| `files:delete` | delete/hide files and folders subject to retention |
|
|
||||||
| `files:admin` | tenant-wide administration of user/group file spaces |
|
|
||||||
|
|
||||||
## Mail Servers
|
|
||||||
|
|
||||||
| Permission | Boundary |
|
|
||||||
|---|---|
|
|
||||||
| `mail_servers:read` | profile metadata and effective policy visibility |
|
|
||||||
| `mail_servers:use` | select an approved profile without reading secrets |
|
|
||||||
| `mail_servers:test` | run server-side connection tests |
|
|
||||||
| `mail_servers:write` | define/edit profiles in allowed scopes |
|
|
||||||
| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or campaign-level credentials where policy allows |
|
|
||||||
|
|
||||||
Reusable encrypted profiles now exist. Effective usability is also constrained by hierarchical mail-profile policy, ownership, allowed/forced profile sets, credential inheritance mode, the lower-level override switch for that mode, and allow/deny patterns.
|
|
||||||
|
|
||||||
## Sessions, API Keys and CSRF
|
|
||||||
|
|
||||||
- Browser login creates an HttpOnly session cookie and a separate readable CSRF cookie.
|
|
||||||
- Unsafe cookie-authenticated requests require matching CSRF cookie/header and stored CSRF hash.
|
|
||||||
- API keys remain supported for CLI/automation and do not use browser CSRF.
|
|
||||||
- Login responses still expose a compatibility session token in the response body; the WebUI does not persist it.
|
|
||||||
|
|
||||||
## Legacy Compatibility
|
|
||||||
|
|
||||||
Runtime aliases remain only for names that are no longer canonical, including:
|
|
||||||
|
|
||||||
```text
|
|
||||||
campaign:write
|
|
||||||
attachments:read
|
|
||||||
attachments:write
|
|
||||||
admin:users
|
|
||||||
admin:users:write
|
|
||||||
admin:api_keys:write
|
|
||||||
admin:settings
|
|
||||||
system:tenants:write
|
|
||||||
system:access:write
|
|
||||||
```
|
|
||||||
|
|
||||||
Canonical scopes are not widened by runtime alias expansion after migration.
|
|
||||||
|
|
||||||
## Deferred Permission Families
|
|
||||||
|
|
||||||
Add these only with their corresponding implemented features:
|
|
||||||
|
|
||||||
```text
|
|
||||||
templates:*
|
|
||||||
address_books:*
|
|
||||||
recipient_lists:*
|
|
||||||
connectors:*
|
|
||||||
dsar:*
|
|
||||||
system:monitoring:read
|
|
||||||
system:backups:run
|
|
||||||
system:backups:restore
|
|
||||||
system:updates:apply
|
|
||||||
system:updates:rollback
|
|
||||||
```
|
|
||||||
|
|
||||||
A separate `retention:*` family is not currently canonical because retention is managed through system settings and tenant policy scopes. Add it only if retention operation duties need separation from general policy/settings administration.
|
|
||||||
@@ -1,8 +1,17 @@
|
|||||||
# GovOPlaN Release Dependencies
|
# GovOPlaN Release Dependencies
|
||||||
|
|
||||||
Release installs must not depend on sibling checkout paths. Local development can keep editable installs and `file:` WebUI links, but release packaging should resolve modules from tagged git refs or from a package registry.
|
This document owns release package composition, signed package catalogs,
|
||||||
|
license checks, catalog publishing, migration baselines, and the final release
|
||||||
|
checklist.
|
||||||
|
|
||||||
## Backend
|
Operator runtime configuration and module install/uninstall execution live in
|
||||||
|
`DEPLOYMENT_OPERATOR_GUIDE.md`.
|
||||||
|
|
||||||
|
## Backend Packages
|
||||||
|
|
||||||
|
Release installs must not depend on sibling checkout paths. Local development
|
||||||
|
can keep editable installs and `file:` WebUI links, but release packaging must
|
||||||
|
resolve modules from tagged git refs or from a package registry.
|
||||||
|
|
||||||
Local development:
|
Local development:
|
||||||
|
|
||||||
@@ -18,47 +27,794 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
`.[server]` is resolved relative to the current working directory. If you create the virtualenv elsewhere, still run the install command from the core checkout:
|
`.[server]` is resolved relative to the current working directory. If you
|
||||||
|
create the virtualenv elsewhere, still run the install command from the core
|
||||||
|
checkout:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
`requirements-release.txt` pins the module repositories to the release tag. Update those refs when cutting a release:
|
`requirements-release.txt` pins the module repositories to the release tag.
|
||||||
|
Update those refs when cutting a release:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.1
|
govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.6
|
||||||
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.1
|
govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.6
|
||||||
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.1
|
govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.6
|
||||||
|
govoplan-organizations git@git.add-ideas.de:add-ideas/govoplan-organizations.git v0.1.6
|
||||||
|
govoplan-identity git@git.add-ideas.de:add-ideas/govoplan-identity.git v0.1.6
|
||||||
|
govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.6
|
||||||
|
govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.6
|
||||||
|
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.6
|
||||||
|
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.6
|
||||||
|
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.6
|
||||||
|
govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.6
|
||||||
```
|
```
|
||||||
|
|
||||||
## WebUI
|
## WebUI Packages
|
||||||
|
|
||||||
Local development uses `webui/package.json`, which may point at sibling module checkouts while active development is happening.
|
Local development uses `webui/package.json`, which may point at sibling module
|
||||||
|
checkouts while active development is happening.
|
||||||
|
|
||||||
Release WebUI installs should use `webui/package.release.json`. It points module dependencies at the same tagged git repositories. To generate a release lockfile, copy it over `package.json` in a release branch or build workspace and then run `npm install` there:
|
Release WebUI installs should use `webui/package.release.json`. It points
|
||||||
|
module dependencies at the same tagged git repositories. After the module tags
|
||||||
|
referenced there exist, generate the committed release lockfile without
|
||||||
|
touching the development package files:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core/webui
|
cd /mnt/DATA/git/govoplan-core
|
||||||
cp package.release.json package.json
|
scripts/generate-release-lock.sh
|
||||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm install
|
cd webui
|
||||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/files-webui`, `@govoplan/mail-webui`, and `@govoplan/campaign-webui` from repository roots even though their source lives below `webui/src`.
|
The module repositories include root-level npm package manifests so git
|
||||||
|
installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`,
|
||||||
|
`@govoplan/files-webui`, `@govoplan/mail-webui`,
|
||||||
|
`@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository
|
||||||
|
roots even though their source lives below `webui/src`.
|
||||||
|
|
||||||
### Release lockfile strategy
|
### Release Lockfile Strategy
|
||||||
|
|
||||||
The supported release composition currently is the full Multi Seal Mail product: core plus files, mail, and campaign. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies.
|
The supported release composition currently is the full GovOPlaN product: core
|
||||||
|
plus access, admin, tenancy, organizations, identity, policy, audit,
|
||||||
|
dashboard, files, mail, campaign, calendar, docs, and ops. Keep one committed
|
||||||
|
full-product release lockfile at
|
||||||
|
`webui/package-lock.release.json`, generated from
|
||||||
|
`webui/package.release.json` in a clean release workspace. Development
|
||||||
|
`package-lock.json` may continue to point at local `file:` dependencies.
|
||||||
|
|
||||||
Frontend module permutations are regression-tested through `GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through committed lockfiles for every possible combination. If a smaller composition becomes a separately shipped product, add an explicit release manifest and lockfile pair for that product, for example `package.release.files-mail.json` and `package-lock.release.files-mail.json`, generated in a clean release workspace from tagged git dependencies.
|
Frontend module permutations are regression-tested through
|
||||||
|
`GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through
|
||||||
|
committed lockfiles for every possible combination. If a smaller composition
|
||||||
|
becomes a separately shipped product, add an explicit release manifest and
|
||||||
|
lockfile pair for that product, for example
|
||||||
|
`package.release.files-mail.json` and `package-lock.release.files-mail.json`,
|
||||||
|
generated in a clean release workspace from tagged git dependencies.
|
||||||
|
|
||||||
|
## Release Tag Script
|
||||||
|
|
||||||
|
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps
|
||||||
|
or accepts the target version, updates Python/WebUI/module manifest versions,
|
||||||
|
commits/tags/pushes the module repositories first, regenerates
|
||||||
|
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
||||||
|
working tree has already been bumped, pass the current version explicitly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/push-release-tag.sh --version 0.1.6
|
||||||
|
```
|
||||||
|
|
||||||
|
`scripts/generate-release-catalog.py` reads installed/discovered
|
||||||
|
`ModuleManifest` objects while writing catalog entries. When a manifest is
|
||||||
|
available, the catalog entry uses the manifest version, points package refs at
|
||||||
|
`v<manifest.version>`, and copies `provides_interfaces` /
|
||||||
|
`requires_interfaces` from the manifest. It also copies module migration order
|
||||||
|
and `migration_tasks` metadata when present. If a manifest cannot be
|
||||||
|
discovered, the entry falls back to the release version passed with `--version`
|
||||||
|
and omits interface and migration-task metadata. This keeps the catalog aligned
|
||||||
|
with independently versioned module packages instead of relying on a hardcoded
|
||||||
|
compatibility table.
|
||||||
|
|
||||||
|
The script also includes GovOPlaN roadmap/scaffold module repositories that do
|
||||||
|
not yet have package metadata. Those repositories are committed, tagged, and
|
||||||
|
pushed with the same release tag, but they are tag-only until they contain
|
||||||
|
`pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories
|
||||||
|
are not listed in `requirements-release.txt` or `webui/package.release.json`.
|
||||||
|
|
||||||
|
Current tag-only module repositories:
|
||||||
|
|
||||||
|
- `govoplan-addresses`
|
||||||
|
- `govoplan-appointments`
|
||||||
|
- `govoplan-cases`
|
||||||
|
- `govoplan-connectors`
|
||||||
|
- `govoplan-dms`
|
||||||
|
- `govoplan-erp`
|
||||||
|
- `govoplan-fit-connect`
|
||||||
|
- `govoplan-forms`
|
||||||
|
- `govoplan-identity-trust`
|
||||||
|
- `govoplan-idm`
|
||||||
|
- `govoplan-ledger`
|
||||||
|
- `govoplan-notifications`
|
||||||
|
- `govoplan-payments`
|
||||||
|
- `govoplan-portal`
|
||||||
|
- `govoplan-reporting`
|
||||||
|
- `govoplan-scheduling`
|
||||||
|
- `govoplan-search`
|
||||||
|
- `govoplan-tasks`
|
||||||
|
- `govoplan-templates`
|
||||||
|
- `govoplan-workflow`
|
||||||
|
- `govoplan-xoev`
|
||||||
|
- `govoplan-xrechnung`
|
||||||
|
- `govoplan-xta-osci`
|
||||||
|
|
||||||
|
## Catalog Trust And Licensing
|
||||||
|
|
||||||
|
GovOPlaN module install and uninstall must remain operator-controlled. The
|
||||||
|
running server may plan and validate package changes, but package mutation is
|
||||||
|
performed by the separate installer daemon or an operator shell during
|
||||||
|
maintenance mode.
|
||||||
|
|
||||||
|
`govoplan-web` is the public static distribution surface for official catalog
|
||||||
|
resources:
|
||||||
|
|
||||||
|
- signed module package catalogs, grouped by release channel
|
||||||
|
- public catalog keyrings
|
||||||
|
- public license verification keyrings
|
||||||
|
- examples and operator-facing download paths
|
||||||
|
|
||||||
|
`govoplan-core` is the verifier and orchestrator:
|
||||||
|
|
||||||
|
- fetches a local or remote module catalog
|
||||||
|
- verifies catalog signatures against configured trusted keys
|
||||||
|
- enforces approved release channels
|
||||||
|
- rejects expired or not-yet-valid catalogs
|
||||||
|
- records accepted catalog sequence numbers for replay protection
|
||||||
|
- checks catalog entry license feature requirements before planning installs
|
||||||
|
- writes installer plans and request records
|
||||||
|
|
||||||
|
Feature and platform modules own their package artifacts, manifests, migration
|
||||||
|
metadata, retirement providers, and optional lifecycle behavior.
|
||||||
|
|
||||||
|
Core accepts either a local catalog file or a remote URL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
||||||
|
```
|
||||||
|
|
||||||
|
If both file and URL are set, the URL wins. The cache is used when a remote
|
||||||
|
fetch fails, so an operator can still inspect the last known catalog. A cached
|
||||||
|
catalog must still pass signature, freshness, channel, and replay validation.
|
||||||
|
|
||||||
|
An official catalog is a JSON object with:
|
||||||
|
|
||||||
|
- `catalog_version`
|
||||||
|
- `channel`
|
||||||
|
- `sequence`
|
||||||
|
- `generated_at`
|
||||||
|
- `not_before` when delayed activation is needed
|
||||||
|
- `expires_at`
|
||||||
|
- `modules`
|
||||||
|
- `signatures`
|
||||||
|
|
||||||
|
Each module entry can declare:
|
||||||
|
|
||||||
|
- backend package name and pinned install reference
|
||||||
|
- WebUI package name and pinned install reference
|
||||||
|
- display metadata and tags
|
||||||
|
- `license_features`, the feature entitlements required to plan that install
|
||||||
|
- `dependencies` and `optional_dependencies`, the module ids expected in the
|
||||||
|
target module set
|
||||||
|
- `migration_safety`, one of `automatic`, `requires_review`, `forward_only`,
|
||||||
|
or `destructive`
|
||||||
|
- `migration_notes`, operator-facing data/migration guidance for review,
|
||||||
|
forward-only, or destructive changes
|
||||||
|
- `migration_after` and `migration_before`, explicit module ids used to order
|
||||||
|
module-owned migration heads when a release needs a live-data sequencing rule
|
||||||
|
- `migration_tasks`, constrained live-data tasks that run around Alembic
|
||||||
|
migration phases. Each task declares `task_id`, `phase`, `summary`,
|
||||||
|
`task_version`, `safety`, `idempotent`, and optionally `timeout_seconds`.
|
||||||
|
The allowed phases are `pre_migration_check`, `pre_migration_prepare`,
|
||||||
|
`post_migration_backfill`, and `post_migration_verify`.
|
||||||
|
- `current_version_min` and `current_version_max_exclusive`, the installed
|
||||||
|
version window from which this catalog target may be applied directly
|
||||||
|
- `bridge_release` and `bridge_notes`, marking a target as an intermediate
|
||||||
|
compatibility release in a staged update path
|
||||||
|
- `allow_downgrade` and `allow_same_version`, explicit opt-ins for reviewed
|
||||||
|
rollback or package-refresh plans
|
||||||
|
- `recovery_tested` and `recovery_notes`, documenting the rehearsal for
|
||||||
|
forward-only or destructive data changes
|
||||||
|
- `provides_interfaces`, named interface contracts exported by this module
|
||||||
|
- `requires_interfaces`, named interface contracts and version ranges required
|
||||||
|
by this module
|
||||||
|
|
||||||
|
The signature is Ed25519 over canonical JSON with both `signature` and
|
||||||
|
`signatures` removed. Core accepts the legacy single `signature` field and the
|
||||||
|
new `signatures` array.
|
||||||
|
When `APP_ENV` is `prod` or `production`, module package catalog signature
|
||||||
|
verification is required by default unless
|
||||||
|
`GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=false` is set explicitly.
|
||||||
|
|
||||||
|
### Independent Module Versions And Interface Ranges
|
||||||
|
|
||||||
|
Modules do not need to ship on the same version number. Release catalogs should
|
||||||
|
pin each package to the exact backend/WebUI ref being installed and declare any
|
||||||
|
cross-module API compatibility through named interfaces.
|
||||||
|
|
||||||
|
Provider shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"provides_interfaces": [
|
||||||
|
{ "name": "files.campaign_attachments", "version": "1.4.0" }
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Requirement shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"requires_interfaces": [
|
||||||
|
{
|
||||||
|
"name": "files.campaign_attachments",
|
||||||
|
"version_min": "1.0.0",
|
||||||
|
"version_max_exclusive": "2.0.0"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
`version_min` is inclusive. `version_max_exclusive` is exclusive, so the range
|
||||||
|
above means `>= 1.0.0` and `< 2.0.0`. Use this for SemVer major-version
|
||||||
|
compatibility lines. Set `"optional": true` only when the module can operate
|
||||||
|
without that interface being present. If a provider is installed but its
|
||||||
|
version is outside the declared optional range, activation is still blocked.
|
||||||
|
|
||||||
|
Catalog validation normalizes these fields and warns when catalog entries do
|
||||||
|
not satisfy each other's ranges. Registry activation and installer preflight
|
||||||
|
perform the blocking checks against the discovered installed manifests before
|
||||||
|
the desired module set is activated.
|
||||||
|
The admin module-management UI shows catalog warnings in the package catalog
|
||||||
|
section and repeats them as warning-level installer preflight issues while a
|
||||||
|
package install is planned.
|
||||||
|
|
||||||
|
Install-plan items carry a `source` field. Manually entered items use
|
||||||
|
`source: "manual"`; entries planned from the package catalog use
|
||||||
|
`source: "catalog"`. Catalog-sourced items also carry a `catalog` metadata
|
||||||
|
object with the validation snapshot used when the item was planned: catalog
|
||||||
|
source/path, source type, cache path, channel, sequence, generated/validity
|
||||||
|
timestamps, signature state, trusted key id, and cache state where available.
|
||||||
|
Catalog provenance changes preflight severity:
|
||||||
|
|
||||||
|
- catalog-sourced installs and updates require a configured, valid package
|
||||||
|
catalog before activation
|
||||||
|
- invalid, untrusted, expired, not-yet-valid, replayed, or unapproved-channel
|
||||||
|
catalogs block catalog-sourced installs and updates
|
||||||
|
- the same catalog validation failures remain warnings for manual install
|
||||||
|
plans, so operators can still use offline or emergency package refs
|
||||||
|
- valid-catalog warnings, such as intentionally unsigned local catalogs when
|
||||||
|
signature enforcement is disabled, remain warnings
|
||||||
|
- selected catalog entries with unsatisfied non-optional named interface ranges
|
||||||
|
block activation before the installer runs
|
||||||
|
- selected catalog entries whose target dependencies are neither installed nor
|
||||||
|
planned block activation before the installer runs
|
||||||
|
- catalog update targets older than the installed module version block unless
|
||||||
|
the catalog entry declares `allow_downgrade: true`
|
||||||
|
- catalog update targets equal to the installed module version block unless the
|
||||||
|
catalog entry declares `allow_same_version: true`
|
||||||
|
- catalog update targets with a `current_version_min` /
|
||||||
|
`current_version_max_exclusive` window block when the installed version is
|
||||||
|
outside that window; publish and apply a bridge release instead
|
||||||
|
- catalog entries marked `forward_only` or `destructive` block activation until
|
||||||
|
the plan row has an explicit data-safety acknowledgement
|
||||||
|
- catalog entries marked `forward_only` or `destructive` also block unless the
|
||||||
|
catalog entry declares `recovery_tested: true` and either the catalog entry or
|
||||||
|
operator plan row contains recovery notes
|
||||||
|
- catalog entries marked `destructive` also require catalog migration notes or
|
||||||
|
operator notes describing the cleanup or retirement plan
|
||||||
|
|
||||||
|
### Update Paths
|
||||||
|
|
||||||
|
Package updates are target-state operations, not one-module-at-a-time runtime
|
||||||
|
toggles. The safe unit of planning is a desired module version set plus a
|
||||||
|
catalog validation snapshot. The installer may install multiple packages into
|
||||||
|
the environment before activation, then validate the discovered manifests and
|
||||||
|
activate the resulting set together.
|
||||||
|
|
||||||
|
Install-plan rows support explicit `install`, `update`, and `uninstall`
|
||||||
|
actions. Catalog planning writes `update` when the module is already installed.
|
||||||
|
Preflight resolves the target set from installed manifests plus the planned
|
||||||
|
catalog entries. Unplanned catalog entries are not treated as installed. When a
|
||||||
|
catalog entry would satisfy a missing dependency or named interface, preflight
|
||||||
|
blocks activation with a companion-update issue; the admin catalog planner adds
|
||||||
|
those companion rows automatically when it can resolve them from the current
|
||||||
|
catalog. The preflight response also includes a structured `target_plan` summary
|
||||||
|
with each planned module's action, current version, catalog target version,
|
||||||
|
package refs, migration-safety level, current-version update window, bridge
|
||||||
|
metadata, recovery metadata, and acknowledgement state.
|
||||||
|
|
||||||
|
Database migrations are planned against that same target module set. When the
|
||||||
|
installer is run with `--migrate`, it calls `govoplan_core.commands.init_db`
|
||||||
|
with the target enabled modules rather than the pre-update startup module list,
|
||||||
|
so newly installed module migration directories are discovered before
|
||||||
|
activation. Preflight also returns a structured migration plan. Its step order is
|
||||||
|
derived from:
|
||||||
|
|
||||||
|
- manifest and catalog `migration_after` / `migration_before` declarations
|
||||||
|
- module dependencies and optional dependencies when both modules are in the
|
||||||
|
target plan
|
||||||
|
- named interface provider/consumer relationships when both sides are in the
|
||||||
|
target plan
|
||||||
|
|
||||||
|
Preflight blocks cycles in that ordering graph. It also blocks non-idempotent
|
||||||
|
module migration tasks, forward-only/destructive tasks without operator
|
||||||
|
acknowledgement, and installed manifest tasks that declare no executor.
|
||||||
|
Catalog-only task executors are marked as pending because they can only be
|
||||||
|
confirmed after the target package is installed. The migrator runs pre-migration
|
||||||
|
tasks, upgrades the ordered module heads first, finishes with Alembic `heads`,
|
||||||
|
and then runs post-migration tasks, so Alembic's revision graph remains
|
||||||
|
authoritative while GovOPlaN still gives operators a module-aware live-data
|
||||||
|
order.
|
||||||
|
|
||||||
|
This avoids circular "upgrade A first / upgrade B first" traps: named interface
|
||||||
|
requirements are solved against the target set, not against each intermediate
|
||||||
|
package-install moment. If the target set cannot satisfy all non-optional
|
||||||
|
interfaces and module dependencies at once, the plan is invalid. Operators
|
||||||
|
should add the necessary module updates to the same plan instead of trying to
|
||||||
|
force an order.
|
||||||
|
|
||||||
|
Live data upgrades need an even stricter rule:
|
||||||
|
|
||||||
|
- migrations must be idempotent and ordered by module migration metadata
|
||||||
|
- destructive schema/data changes need an explicit retirement or cleanup plan,
|
||||||
|
not an automatic package update side effect
|
||||||
|
- cross-module data migrations must be compatible with both the old and target
|
||||||
|
provider interface until activation finishes
|
||||||
|
- rollback must restore the package set and database state together, or be
|
||||||
|
documented as forward-only with a tested recovery procedure
|
||||||
|
- if two modules require mutually incompatible live-data states, the catalog
|
||||||
|
must publish an intermediate compatibility release rather than a circular
|
||||||
|
update chain
|
||||||
|
|
||||||
|
The release catalog is the first safety gate for this. Generated catalogs mark
|
||||||
|
modules with registered migrations as `requires_review` by default. Release
|
||||||
|
authors should keep that value for ordinary reversible migrations, raise it to
|
||||||
|
`forward_only` when database rollback requires restoring a snapshot, and raise
|
||||||
|
it to `destructive` when the update removes or irreversibly rewrites persisted
|
||||||
|
data. Forward-only and destructive entries must include `recovery_tested: true`
|
||||||
|
and recovery notes after a verified restore or forward-recovery rehearsal. The
|
||||||
|
admin install-plan UI exposes the safety level and lets operators record an
|
||||||
|
explicit acknowledgement; preflight keeps acknowledged forward-only/destructive
|
||||||
|
changes visible as warnings.
|
||||||
|
|
||||||
|
In practice, circular dependencies are avoided by designing interfaces with
|
||||||
|
compatibility windows and by publishing bridge releases. A bridge release keeps
|
||||||
|
the old interface while introducing the new one, allowing dependent modules to
|
||||||
|
move first; a later release can retire the old interface after every dependent
|
||||||
|
module has a compatible target version. Use `current_version_min` and
|
||||||
|
`current_version_max_exclusive` to make those direct-update windows explicit in
|
||||||
|
the catalog, and set `bridge_release: true` on intermediate targets that exist
|
||||||
|
primarily to carry installations safely across a compatibility gap.
|
||||||
|
|
||||||
|
Trusted catalog keys are configured locally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":"<base64 public key>"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
For development or tightly controlled deployments, a keyring can be read from a
|
||||||
|
URL and cached:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Production installations should pin the trusted keyring locally or ship it
|
||||||
|
through deployment configuration. Fetching trusted keys from the same public
|
||||||
|
origin as the catalog is convenient, but that origin must not become the only
|
||||||
|
trust root.
|
||||||
|
|
||||||
|
## Dependency Audits
|
||||||
|
|
||||||
|
Dependency vulnerability checks are documented in
|
||||||
|
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same
|
||||||
|
check against release dependency refs on pushes, pull requests, and a weekly
|
||||||
|
schedule.
|
||||||
|
|
||||||
|
Keyring entries support:
|
||||||
|
|
||||||
|
- `key_id`
|
||||||
|
- `public_key` or `public_key_base64`
|
||||||
|
- `status`: `active`, `next`, `retired`, `revoked`, or `disabled`
|
||||||
|
- `not_before`
|
||||||
|
- `not_after`
|
||||||
|
|
||||||
|
Rotation process:
|
||||||
|
|
||||||
|
1. Add the next public key to the local trusted keyring with status `next`.
|
||||||
|
2. Publish catalogs signed by both current and next keys.
|
||||||
|
3. Upgrade installations so the next key is locally trusted.
|
||||||
|
4. Promote the next key to `active`.
|
||||||
|
5. Retire the old key only after every supported installation trusts the new
|
||||||
|
key.
|
||||||
|
6. Mark a compromised key `revoked` and publish a higher sequence catalog
|
||||||
|
signed by an uncompromised key.
|
||||||
|
|
||||||
|
Use replay state in production:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Core records the accepted sequence per channel after a catalog entry is planned
|
||||||
|
from the admin interface. With strict sequence enforcement, a previously
|
||||||
|
accepted sequence is rejected; without strict enforcement, only older sequences
|
||||||
|
are rejected. Catalogs should always expire.
|
||||||
|
|
||||||
|
The sequence state file is operational state, not a trust root. Keep it on
|
||||||
|
persistent storage and include it in normal backups:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"channels": {
|
||||||
|
"stable": {
|
||||||
|
"last_sequence": 42,
|
||||||
|
"accepted_at": "2026-07-07T12:00:00Z",
|
||||||
|
"key_id": "release-key-1",
|
||||||
|
"source": "https://govoplan.example/catalogs/v1/channels/stable.json"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If the file is lost, restore it from backup. If no backup exists, reconstruct
|
||||||
|
each channel from the highest sequence already accepted in installer run
|
||||||
|
records, release records, or the currently deployed module package set. Do not
|
||||||
|
lower `last_sequence` to make an older catalog pass; publish a new higher
|
||||||
|
sequence catalog when the accepted point is uncertain.
|
||||||
|
|
||||||
|
If the file is corrupted, copy it aside for incident review, validate the
|
||||||
|
current signed catalog with channel and freshness enforcement, then rewrite the
|
||||||
|
state with the known accepted sequence. Keep
|
||||||
|
`GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true` and approved-channel
|
||||||
|
checks enabled during recovery. Temporarily disabling
|
||||||
|
`GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE` allows revalidating the same
|
||||||
|
sequence, but older sequences remain rejected once the reconstructed
|
||||||
|
`last_sequence` is in place.
|
||||||
|
|
||||||
|
Approved channels are deployment policy:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
The admin UI can display other catalog metadata, but core rejects catalogs from
|
||||||
|
unapproved channels when validation is configured.
|
||||||
|
|
||||||
|
Catalog entries can require license features:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"license_features": ["module.mail", "support.standard"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Core checks those requirements against an offline license file before allowing
|
||||||
|
the entry into the install plan.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json
|
||||||
|
GOVOPLAN_LICENSE_ENFORCEMENT=true
|
||||||
|
GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
License files are JSON objects with:
|
||||||
|
|
||||||
|
- `license_id`
|
||||||
|
- `subject`
|
||||||
|
- `features`
|
||||||
|
- `valid_from`
|
||||||
|
- `valid_until`
|
||||||
|
- `signature`
|
||||||
|
|
||||||
|
Issue or renew a license from an operator/release shell that has the Ed25519
|
||||||
|
private key:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--issue-license /srv/govoplan/license.json \
|
||||||
|
--license-id customer-2026-07 \
|
||||||
|
--license-subject "Example Municipality" \
|
||||||
|
--license-feature module.mail \
|
||||||
|
--license-feature support.standard \
|
||||||
|
--license-valid-until 2027-07-31T23:59:59Z \
|
||||||
|
--license-signing-key-id license-issuer-1 \
|
||||||
|
--license-signing-private-key /srv/govoplan/secrets/license-issuer-1.pem \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate an imported license without exposing secrets:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--validate-license /srv/govoplan/license.json \
|
||||||
|
--license-trusted-key license-issuer-1="<base64 public key>" \
|
||||||
|
--require-trusted-license \
|
||||||
|
--license-required-feature module.mail \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The CLI and admin module catalog panel report the license id, subject,
|
||||||
|
validity window, signing key id, signed/trusted state, available features, and
|
||||||
|
missing entitlements for the configured package catalog. They do not expose
|
||||||
|
private signing material.
|
||||||
|
|
||||||
|
License enforcement can run in observe-only mode by leaving
|
||||||
|
`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license
|
||||||
|
data is surfaced as a warning but does not block planning.
|
||||||
|
|
||||||
|
Renewal is an ordinary re-issuance with a new `license_id`, extended
|
||||||
|
`valid_until`, and the full intended feature set. Import the renewed JSON to
|
||||||
|
`GOVOPLAN_LICENSE_FILE`, keep the previous file for audit, and validate it
|
||||||
|
before setting enforcement.
|
||||||
|
|
||||||
|
Revocation is handled through the trusted license keyring. Mark a compromised
|
||||||
|
or invalid issuer key as `revoked` or `disabled`, publish or deploy the updated
|
||||||
|
keyring, then reissue affected licenses with an active key. Installations that
|
||||||
|
run with `GOVOPLAN_LICENSE_ENFORCEMENT=true` reject licenses signed only by a
|
||||||
|
revoked key after the local keyring is updated.
|
||||||
|
|
||||||
|
Emergency fallback is deliberately explicit. Operators can temporarily unset
|
||||||
|
`GOVOPLAN_LICENSE_ENFORCEMENT` to keep package planning observable while a
|
||||||
|
license or keyring is recovered. Record the change in the operational incident
|
||||||
|
log, keep catalog signature and channel enforcement enabled, and restore
|
||||||
|
license enforcement after a trusted renewal validates successfully.
|
||||||
|
|
||||||
|
Licensing is intentionally separate from open-source code licensing. The
|
||||||
|
catalog/license mechanism can govern support channels, official release
|
||||||
|
eligibility, hosted update access, professional support, or commercial
|
||||||
|
entitlements without changing the source license of the repositories.
|
||||||
|
|
||||||
|
Production-grade distribution still needs remote registry/git artifact
|
||||||
|
resolution before package-manager apply, a hardened catalog publishing pipeline
|
||||||
|
in `govoplan-web`, and automated key rotation and emergency revocation drills.
|
||||||
|
|
||||||
|
## Release Catalog Publishing
|
||||||
|
|
||||||
|
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
||||||
|
verified by `govoplan-core` before installer plans are accepted. Private signing
|
||||||
|
keys must stay outside all git repositories. Public keyrings are published with
|
||||||
|
the website.
|
||||||
|
|
||||||
|
Create the first catalog signing key on the release machine:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
mkdir -p "$KEY_DIR"
|
||||||
|
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
||||||
|
--key-id release-key-1 \
|
||||||
|
--private-key "$KEY_DIR/release-key-1.pem" \
|
||||||
|
--public-key "$KEY_DIR/release-key-1.pub" \
|
||||||
|
--keyring "$KEY_DIR/catalog-keyring.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
Keep `release-key-1.pem` private. The generated keyring contains only public
|
||||||
|
material.
|
||||||
|
|
||||||
|
Generate the signed catalog into `govoplan-web`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/publish-release-catalog.sh \
|
||||||
|
--version <x.y.z> \
|
||||||
|
--sequence 202607071340 \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web
|
||||||
|
```
|
||||||
|
|
||||||
|
This writes:
|
||||||
|
|
||||||
|
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
||||||
|
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
||||||
|
|
||||||
|
The wrapper validates the catalog with core using the generated public keyring.
|
||||||
|
|
||||||
|
For normal module/core releases, first audit and record migration baselines,
|
||||||
|
then tag and push the module/core repos. Finally publish the website catalog:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/publish-release-catalog.sh \
|
||||||
|
--version <x.y.z> \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web \
|
||||||
|
--commit \
|
||||||
|
--tag \
|
||||||
|
--push
|
||||||
|
```
|
||||||
|
|
||||||
|
The website tag is `catalog-v<x.y.z>`. The public URL is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The public keyring URL is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
||||||
|
tags have been pushed. It runs the migration release audit in automatic mode:
|
||||||
|
warning-only before the first recorded migration baseline, strict after a
|
||||||
|
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
||||||
|
mode explicitly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/push-release-tag.sh \
|
||||||
|
--bump subversion \
|
||||||
|
--strict-migration-audit \
|
||||||
|
--publish-web-catalog \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web-catalog
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--catalog-signing-key` more than once during a key rotation window. The
|
||||||
|
catalog will contain multiple signatures and the public keyring will include the
|
||||||
|
corresponding public keys.
|
||||||
|
|
||||||
|
On a GovOPlaN installation that should consume the official stable catalog:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
For production, copy the public keyring into deployment configuration and pin it
|
||||||
|
locally. Do not rely on a URL-fetched keyring as the only trust root.
|
||||||
|
|
||||||
|
`stable.json` includes a top-level `core_release` section for operator/update
|
||||||
|
tooling. Core is intentionally not listed as a normal module entry because it
|
||||||
|
must not be added to saved enabled-module state. Core upgrades should remain an
|
||||||
|
operator-supervised package update with restart and health checks.
|
||||||
|
|
||||||
|
Key rotation for published catalogs:
|
||||||
|
|
||||||
|
1. Generate the next private key outside git.
|
||||||
|
2. Run `publish-release-catalog.sh` with both signing keys.
|
||||||
|
3. Publish the web catalog/keyring.
|
||||||
|
4. Roll the new public keyring into installations.
|
||||||
|
5. Stop signing with the old key after the supported fleet trusts the new key.
|
||||||
|
6. Mark compromised keys as revoked in the public keyring and publish a higher
|
||||||
|
sequence catalog signed by a trusted uncompromised key.
|
||||||
|
|
||||||
|
## PostgreSQL Release Check
|
||||||
|
|
||||||
|
Release candidates should pass a disposable PostgreSQL migration and startup
|
||||||
|
smoke check before tagging or publishing catalogs. Start the local testbed,
|
||||||
|
then run the permutation check from the core checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
The script checks migrations and `/health` startup for core-only, files-only,
|
||||||
|
mail-only, campaign-only, campaign+files, campaign+mail, and full-product
|
||||||
|
module sets. `--reset-schema` is destructive and must only be used against a
|
||||||
|
throwaway database.
|
||||||
|
|
||||||
|
## Migration Baselines
|
||||||
|
|
||||||
|
Development migrations may be small and numerous while a feature is moving.
|
||||||
|
Before a stable release, unreleased migrations may be rewritten or squashed into
|
||||||
|
a release-level baseline or release-to-release upgrade migration. After a
|
||||||
|
release tag has shipped, released migration revision IDs are immutable.
|
||||||
|
|
||||||
|
The release policy is:
|
||||||
|
|
||||||
|
- unreleased migrations may be folded before release;
|
||||||
|
- released migrations are never rewritten or deleted;
|
||||||
|
- each stable release records the public migration head revisions in
|
||||||
|
`docs/migration-release-baselines.json`;
|
||||||
|
- fresh installations should apply release-level baselines/upgrades, not
|
||||||
|
unreleased create-then-rename churn;
|
||||||
|
- release-to-release schema changes should be folded into one reviewed
|
||||||
|
migration per migration owner where practical.
|
||||||
|
|
||||||
|
Audit the current graph during release preparation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Generate the reviewed/manual squash checklist:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --squash-plan
|
||||||
|
```
|
||||||
|
|
||||||
|
After the release migrations have been reviewed and the graph is final, record
|
||||||
|
the release baseline:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --record-release <x.y.z>
|
||||||
|
```
|
||||||
|
|
||||||
|
Use strict mode to verify that the current heads are recorded:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||||
|
```
|
||||||
|
|
||||||
|
`scripts/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||||
|
non-strict while no release baseline exists, strict after the first baseline is
|
||||||
|
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
||||||
|
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
||||||
|
only for emergency/manual release work.
|
||||||
|
|
||||||
|
Before the first stable release, fold the current development chain into the
|
||||||
|
first public baseline and record that baseline in
|
||||||
|
`docs/migration-release-baselines.json`. The tracking issue is
|
||||||
|
`add-ideas/govoplan-core#223`.
|
||||||
|
|
||||||
|
## Related Operator Documents
|
||||||
|
|
||||||
|
- `DEPLOYMENT_OPERATOR_GUIDE.md`: runtime environment, explicit migrations,
|
||||||
|
backup/restore commands, module installer daemon/supervisor operation, and
|
||||||
|
rollback drills.
|
||||||
|
- `REMOTE_WEBUI_BUNDLES.md`: experimental browser-loaded module bundles for
|
||||||
|
controlled deployments; normal releases use package builds.
|
||||||
|
|
||||||
## Release Checklist
|
## Release Checklist
|
||||||
|
|
||||||
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
||||||
- Tag core, files, mail, and campaign repositories together.
|
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
||||||
- Update `requirements-release.txt` and `webui/package.release.json` when the release tag changes.
|
calendar, and scaffold module repositories together.
|
||||||
- Generate the committed full-product release lockfile from `package.release.json` in a clean build workspace.
|
- Update `requirements-release.txt` and `webui/package.release.json` when the
|
||||||
- Add separate release manifest/lockfile pairs only for module compositions that are shipped as their own products.
|
release tag changes.
|
||||||
|
- Generate the committed full-product release lockfile from
|
||||||
|
`package.release.json` with `scripts/generate-release-lock.sh`.
|
||||||
|
- Run `scripts/release-migration-audit.py --strict` after recording a release
|
||||||
|
baseline.
|
||||||
|
- Run the PostgreSQL release check against a disposable database.
|
||||||
|
- Publish the signed catalog through the release catalog publishing flow above.
|
||||||
|
- Add separate release manifest/lockfile pairs only for module compositions
|
||||||
|
that are shipped as their own products.
|
||||||
- Do not commit local sibling paths into release manifests.
|
- Do not commit local sibling paths into release manifests.
|
||||||
|
|||||||
161
docs/REMOTE_WEBUI_BUNDLES.md
Normal file
161
docs/REMOTE_WEBUI_BUNDLES.md
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
# Remote WebUI Bundle Loading
|
||||||
|
|
||||||
|
GovOPlaN WebUI modules normally ship through the core WebUI package graph:
|
||||||
|
install a tagged npm/git dependency, run `npm install`, rebuild the shell, and
|
||||||
|
restart or reload the served assets. Remote WebUI bundles are an experimental
|
||||||
|
future path for controlled deployments where a backend-enabled module is not in
|
||||||
|
the local WebUI package graph and the operator wants the shell to load its
|
||||||
|
frontend without rebuilding core.
|
||||||
|
|
||||||
|
This design defines the target guardrails. The current rebuild/reload path
|
||||||
|
remains the default production path.
|
||||||
|
|
||||||
|
## Current Rebuild Path
|
||||||
|
|
||||||
|
The supported release path is:
|
||||||
|
|
||||||
|
1. Install or remove backend and WebUI package dependencies through the trusted
|
||||||
|
installer CLI/daemon.
|
||||||
|
2. Snapshot package state before mutation.
|
||||||
|
3. Run `npm install` and optionally `npm run build`.
|
||||||
|
4. Restart or reload the served WebUI assets.
|
||||||
|
5. Use installer rollback if package apply, restart, or health checks fail.
|
||||||
|
|
||||||
|
Strengths:
|
||||||
|
|
||||||
|
- package manager and lockfile semantics stay conventional
|
||||||
|
- CSP can stay strict because all code is served as built assets
|
||||||
|
- rollback restores package files and lockfiles
|
||||||
|
- local development uses sibling workspace dependencies naturally
|
||||||
|
|
||||||
|
Costs:
|
||||||
|
|
||||||
|
- frontend changes require a rebuild/reload
|
||||||
|
- hot enabling a module with frontend code is not possible in the running shell
|
||||||
|
- failed rebuilds happen at install time, not at lazy module-load time
|
||||||
|
|
||||||
|
## Remote Bundle Path
|
||||||
|
|
||||||
|
Remote loading is only for modules that are enabled by the backend and absent
|
||||||
|
from `virtual:govoplan-installed-modules`. The backend manifest exposes:
|
||||||
|
|
||||||
|
- `FrontendModule.asset_manifest`
|
||||||
|
- `asset_manifest_integrity`
|
||||||
|
- `asset_manifest_signature`
|
||||||
|
- `asset_manifest_public_key_id`
|
||||||
|
- `asset_manifest_contract_version`
|
||||||
|
|
||||||
|
The WebUI shell fetches the asset manifest, verifies manifest integrity and/or
|
||||||
|
signature, validates the manifest contract, fetches the entry bundle, verifies
|
||||||
|
the entry integrity, imports the bundle, validates the exported
|
||||||
|
`PlatformWebModule`, and applies backend metadata before registering routes,
|
||||||
|
navigation, and UI capabilities.
|
||||||
|
|
||||||
|
Unsigned and unhashed manifests are skipped. Entries without integrity are
|
||||||
|
skipped. A module id mismatch is skipped.
|
||||||
|
|
||||||
|
## Asset Manifest Contract
|
||||||
|
|
||||||
|
Contract version `1`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"contractVersion": "1",
|
||||||
|
"moduleId": "files",
|
||||||
|
"entry": "./files-webui.remote.js",
|
||||||
|
"entryIntegrity": "SHA-256-<base64-or-hex-digest>",
|
||||||
|
"moduleExport": "default"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The backend manifest carries the manifest-level trust metadata. The remote
|
||||||
|
asset manifest carries the concrete entry URL and entry digest.
|
||||||
|
|
||||||
|
## Compatibility Checks
|
||||||
|
|
||||||
|
Before a remote bundle is accepted:
|
||||||
|
|
||||||
|
- backend module must be enabled
|
||||||
|
- local WebUI module with the same id must be absent
|
||||||
|
- manifest contract must be supported by the shell
|
||||||
|
- manifest `moduleId`, exported module `id`, and backend module id must match
|
||||||
|
- backend metadata remains authoritative for label, version, dependencies,
|
||||||
|
nav, and runtime UI capability exposure
|
||||||
|
- future contract versions must declare required shell capabilities so old
|
||||||
|
shells fail closed
|
||||||
|
|
||||||
|
Remote bundles must not broaden backend permissions. They can only contribute
|
||||||
|
routes/nav/capabilities that the backend metadata and existing permission
|
||||||
|
checks allow.
|
||||||
|
|
||||||
|
## CSP
|
||||||
|
|
||||||
|
The current implementation imports verified entry bytes through a blob URL. A
|
||||||
|
production CSP for this mode must explicitly allow:
|
||||||
|
|
||||||
|
- `connect-src` for the approved asset-manifest and bundle origins
|
||||||
|
- `script-src` for `blob:` only when remote loading is enabled
|
||||||
|
- no `unsafe-inline` requirement for remote modules
|
||||||
|
|
||||||
|
If an installation cannot allow `blob:` scripts, the follow-up implementation
|
||||||
|
should use signed same-origin module assets with static URLs instead of blob
|
||||||
|
imports. Remote loading must be disableable by configuration so strict-CSP
|
||||||
|
deployments can keep the rebuild path only.
|
||||||
|
|
||||||
|
## Cache Invalidation
|
||||||
|
|
||||||
|
The shell cache key is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<module-id>:<asset-manifest-url>:<backend-module-version>
|
||||||
|
```
|
||||||
|
|
||||||
|
Release catalogs should publish immutable manifest and entry URLs or change at
|
||||||
|
least one cache-key component on every release. Emergency rollback can point the
|
||||||
|
backend manifest at a previous immutable manifest URL or lower the enabled
|
||||||
|
module version after the backend package rollback.
|
||||||
|
|
||||||
|
Browsers may still cache remote responses. Approved asset servers should use:
|
||||||
|
|
||||||
|
- long cache lifetimes only for content-addressed immutable assets
|
||||||
|
- short cache lifetimes or explicit revalidation for channel/latest manifest
|
||||||
|
aliases
|
||||||
|
- `Cache-Control: no-store` for emergency override manifests
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Remote WebUI rollback is metadata rollback, not package-manager rollback:
|
||||||
|
|
||||||
|
- if the backend package install rolls back, the previous backend frontend
|
||||||
|
metadata returns
|
||||||
|
- if only remote assets are bad, publish a new manifest URL or revert the
|
||||||
|
backend/frontend metadata to the last known good manifest
|
||||||
|
- failed remote loading must degrade by omitting the remote module frontend,
|
||||||
|
not by breaking the shell
|
||||||
|
|
||||||
|
Installer run records should eventually include remote asset manifest URLs,
|
||||||
|
integrity, signature key ids, and load-test results when a plan enables a
|
||||||
|
remote frontend.
|
||||||
|
|
||||||
|
## Local And Development Behavior
|
||||||
|
|
||||||
|
Local development should keep using workspace/file dependencies and Vite. Remote
|
||||||
|
loading is useful for integration testing release artifacts, not for day-to-day
|
||||||
|
module UI development.
|
||||||
|
|
||||||
|
Development deployments may use unsigned catalogs and local asset servers only
|
||||||
|
when signature/integrity enforcement is intentionally disabled. The remote
|
||||||
|
loader itself still requires an integrity hash or a verifiable signature.
|
||||||
|
|
||||||
|
## Follow-Up Slices
|
||||||
|
|
||||||
|
1. Add a server-side remote-bundle policy flag and surface whether remote
|
||||||
|
loading is enabled in platform metadata.
|
||||||
|
2. Add CSP documentation/config generation for strict rebuild-only mode versus
|
||||||
|
controlled remote-bundle mode.
|
||||||
|
3. Add an installer/catalog preflight that validates remote WebUI asset
|
||||||
|
manifests and records verified identity in installer run records.
|
||||||
|
4. Add Playwright coverage for a signed test remote bundle, failed digest, bad
|
||||||
|
module id, cache-key refresh, and fallback when the module is unavailable.
|
||||||
|
5. Add release tooling to emit immutable remote asset manifests with digest,
|
||||||
|
signature, key id, and rollback metadata.
|
||||||
80
docs/SELF_HOSTED_INSTALLABILITY.md
Normal file
80
docs/SELF_HOSTED_INSTALLABILITY.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Self-Hosted Installability
|
||||||
|
|
||||||
|
GovOPlaN uses a staged self-hosted installability path.
|
||||||
|
|
||||||
|
## Packaging Decision
|
||||||
|
|
||||||
|
The early packaging approach is a staged combination:
|
||||||
|
|
||||||
|
1. Generate an explicit environment template and validate it before startup.
|
||||||
|
2. Use a Compose-backed production-like development profile for local rehearsal.
|
||||||
|
3. Use the deployment operator guide as the runbook for migrations, workers,
|
||||||
|
backups, health checks, and module installer rollback drills.
|
||||||
|
4. Use the module installer CLI/daemon for package mutation once the runtime is
|
||||||
|
already installed and under maintenance mode.
|
||||||
|
|
||||||
|
This keeps first installation understandable while still preserving the later
|
||||||
|
goal of install/update/uninstall through signed catalogs and the installer
|
||||||
|
daemon. The API server must not run package managers from request handlers.
|
||||||
|
|
||||||
|
## Config Bootstrap
|
||||||
|
|
||||||
|
Generate a self-hosted template:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||||
|
--profile self-hosted \
|
||||||
|
--generate-secrets \
|
||||||
|
--output .env.self-hosted
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate the current shell environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
set -a
|
||||||
|
. .env.self-hosted
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||||
|
```
|
||||||
|
|
||||||
|
The command reports all known blockers at once. Production-like/self-hosted
|
||||||
|
profiles require explicit `APP_ENV`, `DATABASE_URL`, `MASTER_KEY_B64`,
|
||||||
|
`ENABLED_MODULES`, and `CORS_ORIGINS`. Production rejects SQLite, development
|
||||||
|
bootstrap, insecure auth cookies, and unsigned catalog trust roots when a
|
||||||
|
catalog source is configured.
|
||||||
|
|
||||||
|
## Production-Like Dev Stack
|
||||||
|
|
||||||
|
Use the local production-like wrapper for repeatable rehearsal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/production-like-dev.sh validate-config
|
||||||
|
scripts/production-like-dev.sh seed
|
||||||
|
scripts/production-like-dev.sh start
|
||||||
|
```
|
||||||
|
|
||||||
|
Stop Docker dependencies:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/production-like-dev.sh stop
|
||||||
|
```
|
||||||
|
|
||||||
|
Reset all profile data:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/production-like-dev.sh reset --yes
|
||||||
|
```
|
||||||
|
|
||||||
|
The start command delegates to `scripts/launch-production-like-dev.sh`, which
|
||||||
|
runs API, worker, and WebUI in the foreground. Stop those processes with
|
||||||
|
`Ctrl+C` in the launcher terminal.
|
||||||
|
|
||||||
|
## Module Boundary Gate
|
||||||
|
|
||||||
|
`scripts/check_dependency_boundaries.py` is part of the focused verification
|
||||||
|
path. It checks backend imports and WebUI package/source imports so modules do
|
||||||
|
not grow hidden runtime dependencies on each other. Feature modules should
|
||||||
|
integrate through core capabilities, backend APIs/events, route contributions,
|
||||||
|
or explicit UI extension points.
|
||||||
216
docs/UI_UX_DECISION_LEDGER.md
Normal file
216
docs/UI_UX_DECISION_LEDGER.md
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
# GovOPlaN UI/UX Decision Ledger
|
||||||
|
|
||||||
|
This ledger records product UI/UX decisions that affect admin, settings,
|
||||||
|
configuration, connector, policy, and module-management surfaces. It is a
|
||||||
|
binding design reference: future implementation should follow these decisions
|
||||||
|
unless the decision is explicitly revised here and affected screens are updated
|
||||||
|
to match.
|
||||||
|
|
||||||
|
Active tracking issue: `add-ideas/govoplan-core#225`.
|
||||||
|
|
||||||
|
## Operating Rule
|
||||||
|
|
||||||
|
GovOPlaN must expose advanced platform capability without presenting the user
|
||||||
|
with every option at once. Non-technical users should be able to complete common
|
||||||
|
workflows through guided, plain-language flows. Expert and diagnostic detail may
|
||||||
|
exist, but it must be deliberately layered.
|
||||||
|
|
||||||
|
The ethical design doctrine in `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` is the
|
||||||
|
normative baseline for this ledger. A screen can be visually quiet and still be
|
||||||
|
ethically complete only when it preserves context, consequence,
|
||||||
|
contestability, responsibility, and traceability at the point of action.
|
||||||
|
|
||||||
|
## Binding Decisions
|
||||||
|
|
||||||
|
| ID | Decision | Status | Applies To |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| UX-001 | Use progressive disclosure by default. Common decisions stay visible; advanced or hazardous options live in collapsed panels, later wizard steps, or explicit advanced sections. | Accepted | Admin, settings, connector setup, policy editors, module operations |
|
||||||
|
| UX-002 | Raw JSON is not a primary configuration editor. Every normal configuration path needs typed controls, validation, and help text. JSON may be shown for import/export, diagnostics, or expert inspection only. | Accepted | All admin/configuration UIs |
|
||||||
|
| UX-003 | Prefer guided workflows over option dumps for setup and risky changes. Use wizards for connector setup, configuration package import, module install/uninstall, destructive actions, and policy changes with broad impact. | Accepted | Connectors, package import, module lifecycle, governance/policy |
|
||||||
|
| UX-004 | Discover technical values when possible. Ask for the smallest user-known input, then discover and prefill technical fields for review. | Accepted | File connectors, mail/groupware, public URLs, future external providers |
|
||||||
|
| UX-005 | Disabled actions and failed steps must explain why they are unavailable, who can fix them, and where to go next. Silent disabled states are not acceptable for primary actions. | Accepted | All primary actions |
|
||||||
|
| UX-006 | Explanations must be available but quiet. Use short inline text and expose richer explanations through help affordances, side panels, expandable sections, or review steps. | Accepted | All complex forms and flows |
|
||||||
|
| UX-007 | Creation/editing should prefer modals or focused step flows when it reduces page clutter. Overview and comparison screens remain full-page. | Accepted | Settings/admin surfaces |
|
||||||
|
| UX-008 | Similar concepts must use shared placement and components: server/credential/policy rows, problem lists, review steps, advanced panels, confirmation modals, and empty/error states. | Accepted | Core WebUI and module WebUIs |
|
||||||
|
| UX-009 | Preflight and diagnostics are product UX. Validation, policy, permission, dependency, and capability failures must be written for operators before exposing internal details. | Accepted | Installer, connectors, policy, package import |
|
||||||
|
| UX-010 | Context, decision, and consequence must be visible together for actions that affect rights, duties, records, money, communication, retention, external systems, or workflow state. | Accepted | Workflow, admin, portal, policy, connector, records, payments |
|
||||||
|
| UX-011 | Navigation is not consent. Route changes, panel switches, and passive selection must not execute consequential actions without an explicit action surface. | Accepted | All WebUI surfaces |
|
||||||
|
| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit |
|
||||||
|
| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records |
|
||||||
|
| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments |
|
||||||
|
|
||||||
|
## Confirmed Implementation Decisions
|
||||||
|
|
||||||
|
These decisions were accepted on 2026-07-09 for the first implementation slice.
|
||||||
|
They shape reusable components and screen structure. Future changes must revise
|
||||||
|
this section and update affected screens.
|
||||||
|
|
||||||
|
### DUE-001: Primary Admin Configuration Shell
|
||||||
|
|
||||||
|
Decision: admin/configuration surfaces should standardize on a two-zone layout.
|
||||||
|
|
||||||
|
- Left or top area: searchable overview/list, status, and primary actions.
|
||||||
|
- Main area: selected item summary and common settings.
|
||||||
|
- Modal/wizard: create, connect, edit, test, review, and confirm actions.
|
||||||
|
- Collapsed advanced panels: rarely used technical fields.
|
||||||
|
|
||||||
|
Use this for file connectors and mail servers first, then migrate policy,
|
||||||
|
retention, API keys, and module operations.
|
||||||
|
|
||||||
|
### DUE-002: Wizard Step Model
|
||||||
|
|
||||||
|
Decision: standard wizard steps and names use this baseline:
|
||||||
|
|
||||||
|
1. Choose type or scope.
|
||||||
|
2. Enter essentials.
|
||||||
|
3. Discover or test.
|
||||||
|
4. Configure ownership and policy.
|
||||||
|
5. Review changes and blockers.
|
||||||
|
6. Save or submit for operator action.
|
||||||
|
|
||||||
|
Not every wizard needs every step, but flows should use these names and order
|
||||||
|
where applicable.
|
||||||
|
|
||||||
|
This applies to explicit assisted setup, onboarding, import, preflight, and
|
||||||
|
risky multi-step operations. It is not the default shape for ordinary create or
|
||||||
|
edit dialogs.
|
||||||
|
|
||||||
|
### DUE-003: Explanation Placement
|
||||||
|
|
||||||
|
Decision: richer explanations should use this placement model:
|
||||||
|
|
||||||
|
- Short helper text below labels only when it prevents common mistakes.
|
||||||
|
- Tooltips for icon-only controls and compact terms.
|
||||||
|
- Expandable "Why?" or "Details" blocks for contextual explanations.
|
||||||
|
- Right-side detail panel or review step for preflight/provenance/diagnostics.
|
||||||
|
|
||||||
|
Avoid permanently visible paragraphs inside dense admin cards.
|
||||||
|
|
||||||
|
### DUE-004: Advanced Options Contract
|
||||||
|
|
||||||
|
Decision: advanced options are fields that are needed for control, compatibility,
|
||||||
|
or diagnostics but are not part of the common setup path.
|
||||||
|
|
||||||
|
- protocol-specific endpoints, ports, path overrides, TLS/signing toggles,
|
||||||
|
timeout/retry tuning, raw headers, migration/destructive flags, and fallback
|
||||||
|
compatibility settings are advanced.
|
||||||
|
- names, descriptions, provider type, base URL, ownership, policy mode, and
|
||||||
|
basic credentials are not advanced.
|
||||||
|
|
||||||
|
Advanced fields must still be editable through typed controls.
|
||||||
|
|
||||||
|
### DUE-005: Blocker Language Contract
|
||||||
|
|
||||||
|
Decision: all disabled or blocked primary actions should use a shared structured
|
||||||
|
reason object.
|
||||||
|
|
||||||
|
Shape:
|
||||||
|
|
||||||
|
- `summary`: short plain-language reason.
|
||||||
|
- `details`: optional explanation.
|
||||||
|
- `required_action`: what needs to happen.
|
||||||
|
- `actor`: who can do it, for example system administrator or tenant admin.
|
||||||
|
- `target`: where to go or which setting/capability is missing.
|
||||||
|
- `technical_details`: optional expandable developer/operator data.
|
||||||
|
|
||||||
|
For simple missing required fields, highlight the fields and put the structured
|
||||||
|
reason in a compact hover/focus bubble on the disabled primary action instead of
|
||||||
|
adding a large persistent warning block.
|
||||||
|
|
||||||
|
### DUE-006: First Migration Surface
|
||||||
|
|
||||||
|
Decision: convert file connectors first, then mail servers. They share the same
|
||||||
|
server/credential/policy model, are high-value, and will prove the reusable
|
||||||
|
patterns quickly.
|
||||||
|
|
||||||
|
### DUE-007: Adaptive Create/Edit Forms
|
||||||
|
|
||||||
|
Decision: ordinary create/edit dialogs should show the full editable state in an
|
||||||
|
adaptive form, not force a linear wizard.
|
||||||
|
|
||||||
|
- The user chooses a type, provider, credential mode, or policy mode.
|
||||||
|
- The dialog immediately adjusts to show only the fields relevant to that state.
|
||||||
|
- Editing an existing object uses the same field grouping and layout as creating
|
||||||
|
it, so users can recognize the state they configured.
|
||||||
|
- Discovery and test actions must give visible feedback in the same dialog:
|
||||||
|
directly usable, discovered alternative, credentials needed/rejected, or not
|
||||||
|
found with the next action the user can take.
|
||||||
|
- Wizard shells remain available for assisted setup, first-run guidance,
|
||||||
|
imports, discovery-heavy flows, and operational preflight workflows.
|
||||||
|
|
||||||
|
## Implementation Sequence
|
||||||
|
|
||||||
|
| Phase | Scope | Output |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| 0 | UX inventory | List every admin/settings/configuration surface, classify it, and record whether it violates a binding decision. |
|
||||||
|
| 1 | Core primitives | Shared wizard shell, advanced panel, help affordance, blocker callout, problem list, review step, and discovery/test result components. |
|
||||||
|
| 2 | File connectors | Adaptive create/edit for provider/server, credential, discovery/test, ownership/policy, plus optional assisted wizard later. |
|
||||||
|
| 3 | Mail servers | Same adaptive pattern as files, adapted to server/credential/policy and test-send/test-login behavior. |
|
||||||
|
| 4 | Policy/retention editors | Effective value first, provenance visible, override/edit in modal, blocked edits explained. |
|
||||||
|
| 5 | Module/package operations | Step-based install/uninstall flow with preflight, maintenance, daemon handoff, migration, and rollback explanation. |
|
||||||
|
| 6 | Remaining settings/admin screens | Apply inventory findings by priority and remove one-off layouts. |
|
||||||
|
|
||||||
|
## Current Surface Inventory
|
||||||
|
|
||||||
|
This is the phase-0 inventory baseline. It should be extended as each screen is
|
||||||
|
converted or reviewed.
|
||||||
|
|
||||||
|
| Surface | Repository | UX State | Next Action |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| File connector settings | `govoplan-files` | First adaptive modal slice started: connections and credentials now use full-state create/edit forms with conditional fields, advanced panels, and blocker primitives. Wizard shell is retained for later assisted setup. Central policy card still needs a layered editor. | Finish provider discovery/test-in-flow, then convert policy editing. |
|
||||||
|
| Mail server settings | `govoplan-mail` / `govoplan-core` | Uses the shared server/credential model visually, but create/edit still needs the same adaptive pattern as files. | Migrate to adaptive server/credential/policy dialogs, with optional assisted wizard later. |
|
||||||
|
| Connector policy/effective rows | `govoplan-core`, module UIs | Effective-policy direction exists, but many editors still expose broad option sets. | Put effective value first, move overrides into modal, and explain blocked edits. |
|
||||||
|
| Admin module management | `govoplan-admin` | Has preflight concepts, but operational choices are still technical and dense. | Convert install/uninstall/package changes to operator wizards. |
|
||||||
|
| Configuration packages | `govoplan-admin` | Catalog/import work exists, but package editing can still drift toward technical fields. | Add guided import/review/problem-list flow. |
|
||||||
|
| Retention and privacy | `govoplan-core` | Functional editor exists; consequence language and provenance can be stronger. | Layer advanced retention options and add review for broad changes. |
|
||||||
|
| API keys | `govoplan-access` / admin UI | Security-sensitive creation needs least-privilege guidance. | Add scoped creation wizard with expiry/owner review. |
|
||||||
|
| User settings | `govoplan-core` | Preferences persistence exists; interface navigation issue was fixed earlier, but the surface still needs UX review. | Keep simple sections, remove double-click traps, and add quiet explanations. |
|
||||||
|
|
||||||
|
## Impact Index
|
||||||
|
|
||||||
|
| Surface | Current Risk | Expected Pattern |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| File connectors | Too many technical fields and unclear setup order. | Adaptive create/edit form with optional assisted wizard, discovery/test, credential binding, policy review, and advanced protocol panel. |
|
||||||
|
| Mail servers | Similar server/credential/policy concepts risk diverging from files. | Same tree/list and adaptive server/credential/policy model as file connectors. |
|
||||||
|
| Connector credentials | Security-sensitive details can overwhelm users. | Separate credential flow with secret-reference language and test result explanation. |
|
||||||
|
| Policy and effective settings | Users need to know why a value is inherited or locked. | Effective row first, source/provenance, local override action, actionable blocked reason. |
|
||||||
|
| Module install/uninstall | Operationally risky, currently inherently technical. | Operator wizard with preflight, maintenance, daemon handoff, review, and rollback explanation. |
|
||||||
|
| Configuration packages | Could become package JSON editing. | Package catalog/import wizard using provider data requirements and problem lists. |
|
||||||
|
| Retention/privacy | High-risk settings need explanation and provenance. | Layered editor with plain-language consequences and review. |
|
||||||
|
| Automation/workflow commands | Hidden side effects would undermine accountability. | Action/effect preview, system-actor display, command record, retry/quarantine/manual states, and audit links. |
|
||||||
|
| Postbox and encrypted communication | Retraction and access can be misunderstood. | Honest key-fetch/decryption state, expiry limits, recipient/device access provenance, and delivery evidence. |
|
||||||
|
| API keys | Security-sensitive creation and scope selection. | Scoped creation wizard, least-privilege suggestions, clear expiry/owner explanation. |
|
||||||
|
| User settings | Needs clarity and persistence across profile/interface/preferences. | Simple settings sections with immediate feedback and no double-click navigation traps. |
|
||||||
|
|
||||||
|
## Review Checklist
|
||||||
|
|
||||||
|
Every new or changed admin/configuration surface should answer:
|
||||||
|
|
||||||
|
- What is the common path, and is it visible without noise?
|
||||||
|
- Which fields are advanced, and are they collapsed by default?
|
||||||
|
- Is every configuration value editable through typed controls?
|
||||||
|
- Does the flow avoid JSON as the primary editor?
|
||||||
|
- Does the screen explain disabled actions and failed validation in plain
|
||||||
|
language?
|
||||||
|
- Does it say who can fix a blocker and where?
|
||||||
|
- Does it reuse existing core patterns for wizard steps, problem lists, modals,
|
||||||
|
help, and review?
|
||||||
|
- Is there a review or preflight step before broad, destructive, or risky
|
||||||
|
changes?
|
||||||
|
- Does the action surface show consequence, reversibility, and audit evidence
|
||||||
|
when rights, duties, records, money, communication, external systems, or
|
||||||
|
workflow state are affected?
|
||||||
|
- If automation is involved, can the user see the trigger, system actor,
|
||||||
|
observed effects, and failure/manual-intervention state?
|
||||||
|
- Are technical details available without being the first thing the user sees?
|
||||||
|
|
||||||
|
## Revision Rule
|
||||||
|
|
||||||
|
When a UX decision changes:
|
||||||
|
|
||||||
|
1. Update this ledger.
|
||||||
|
2. Update the affected shared components.
|
||||||
|
3. Update existing screens listed in the impact index.
|
||||||
|
4. Add or update Gitea issues for any remaining surfaces that still follow the
|
||||||
|
old decision.
|
||||||
|
5. Sync the wiki.
|
||||||
71
docs/audits/2026-07-09-dependency-audit.md
Normal file
71
docs/audits/2026-07-09-dependency-audit.md
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
# Dependency Audit - 2026-07-09
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Status: remediated.
|
||||||
|
|
||||||
|
Initial result:
|
||||||
|
|
||||||
|
- Python audit failed: 24 advisories were reported across `cryptography`,
|
||||||
|
`pip`, `python-multipart`, `pyzipper`, and `starlette`.
|
||||||
|
- npm production audit passed: `npm audit --omit=dev` reported 0
|
||||||
|
vulnerabilities.
|
||||||
|
|
||||||
|
Python findings:
|
||||||
|
|
||||||
|
| Package | Installed | Advisory count | Minimum reported fix |
|
||||||
|
| --- | ---: | ---: | --- |
|
||||||
|
| `cryptography` | `44.0.0` | 5 | `48.0.1` |
|
||||||
|
| `pip` | `26.0.1` | 3 | `26.1.2` |
|
||||||
|
| `python-multipart` | `0.0.17` | 7 | `0.0.31` |
|
||||||
|
| `pyzipper` | `0.3.6` | 1 | `0.4.0` |
|
||||||
|
| `starlette` | `0.41.3` | 8 | `1.3.1` |
|
||||||
|
|
||||||
|
Private GovOPlaN packages were skipped by `pip-audit` because they are not
|
||||||
|
published on PyPI. That is expected for local editable development installs.
|
||||||
|
|
||||||
|
The remediation below upgrades those dependencies and records the compatibility
|
||||||
|
checks run against core, files, and campaign behavior.
|
||||||
|
|
||||||
|
## Remediation
|
||||||
|
|
||||||
|
Remediation applied on 2026-07-09:
|
||||||
|
|
||||||
|
- upgraded core's FastAPI floor to `fastapi>=0.139,<1`, resolving Starlette to
|
||||||
|
`starlette==1.3.1`
|
||||||
|
- upgraded core's cryptography floor to `cryptography>=48.0.1,<50`, resolving
|
||||||
|
to `cryptography==49.0.0`
|
||||||
|
- declared the files module upload parser dependency as
|
||||||
|
`python-multipart>=0.0.31,<1`, resolving to `python-multipart==0.0.32`
|
||||||
|
- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to
|
||||||
|
`pyzipper==0.4.0`
|
||||||
|
- upgraded the local audit environment to `pip==26.1.2`
|
||||||
|
- removed the obsolete local `govoplan-module-multimailer` editable install
|
||||||
|
from the audit environment so the audit reflects the split module product
|
||||||
|
|
||||||
|
Post-remediation result:
|
||||||
|
|
||||||
|
- `bash scripts/check-dependency-audits.sh`: passed, no known Python
|
||||||
|
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
||||||
|
- `python -m pip check`: passed.
|
||||||
|
- `bash scripts/check-module-matrix.sh`: passed.
|
||||||
|
- `python -m unittest tests.test_api_smoke`: passed.
|
||||||
|
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- `pip-audit` still reports private GovOPlaN packages as skipped because they
|
||||||
|
are not published on PyPI. That is expected for editable local development
|
||||||
|
installs.
|
||||||
|
- `httpx2>=2.5,<3` is included in development requirements so Starlette's
|
||||||
|
`TestClient` uses the non-deprecated backend. `httpx==0.28.1` remains in dev
|
||||||
|
requirements for tests that mock connector HTTP responses directly.
|
||||||
|
- Split-module API routers now use Starlette's renamed
|
||||||
|
`HTTP_422_UNPROCESSABLE_CONTENT` status constant. This preserves the 422
|
||||||
|
status code while avoiding the deprecated `HTTP_422_UNPROCESSABLE_ENTITY`
|
||||||
|
alias.
|
||||||
46
docs/configuration-package-catalog.example.json
Normal file
46
docs/configuration-package-catalog.example.json
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"channel": "stable",
|
||||||
|
"packages": [
|
||||||
|
{
|
||||||
|
"package_id": "govoplan.application-handling.basic",
|
||||||
|
"name": "Basic application handling",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"description": "Portal form, case workflow, task creation, mail notification, payment setup, and access roles for a simple application process.",
|
||||||
|
"publisher": "ADD ideas",
|
||||||
|
"category": "workflow",
|
||||||
|
"artifact_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-configuration-packages.git@application-handling-basic-v0.1.0",
|
||||||
|
"artifact_sha256": "<sha256>",
|
||||||
|
"required_modules": [
|
||||||
|
{ "module_id": "portal", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "forms", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "cases", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "workflow", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "tasks", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "templates", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "mail", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "payments", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "access", "version": ">=0.1.0" },
|
||||||
|
{ "module_id": "audit", "version": ">=0.1.0" }
|
||||||
|
],
|
||||||
|
"required_capabilities": [
|
||||||
|
"configuration.provider",
|
||||||
|
"access.configuration",
|
||||||
|
"forms.configuration",
|
||||||
|
"workflow.configuration",
|
||||||
|
"mail.configuration",
|
||||||
|
"payments.configuration"
|
||||||
|
],
|
||||||
|
"tags": ["official", "example", "public-service"],
|
||||||
|
"signature": {
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": "configuration-release-key-1",
|
||||||
|
"value": "<base64 signature over the package artifact metadata>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"signature": {
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": "configuration-catalog-key-1",
|
||||||
|
"value": "<base64 signature over this JSON object without the signature field>"
|
||||||
|
}
|
||||||
|
}
|
||||||
416
docs/gitea-labels.json
Normal file
416
docs/gitea-labels.json
Normal file
@@ -0,0 +1,416 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "type/bug",
|
||||||
|
"color": "d73a4a",
|
||||||
|
"description": "A reproducible defect, regression, or incorrect behavior.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "type/feature",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "New user-visible behavior or platform capability.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "type/task",
|
||||||
|
"color": "1d76db",
|
||||||
|
"description": "Implementation, maintenance, migration, or operational work.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "type/debt",
|
||||||
|
"color": "fbca04",
|
||||||
|
"description": "Cleanup, refactoring, risk reduction, or deferred engineering work.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "type/docs",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "Documentation, process, or developer workflow work.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "priority/p0",
|
||||||
|
"color": "b60205",
|
||||||
|
"description": "Immediate stop-the-line priority.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "priority/p1",
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "High priority for the next focused work window.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "priority/p2",
|
||||||
|
"color": "fbca04",
|
||||||
|
"description": "Normal planned priority.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "priority/p3",
|
||||||
|
"color": "c2e0c6",
|
||||||
|
"description": "Low priority or opportunistic cleanup.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "status/triage",
|
||||||
|
"color": "d4c5f9",
|
||||||
|
"description": "Needs review, ownership, priority, or acceptance criteria.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "status/ready",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "Ready for implementation.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "status/in-progress",
|
||||||
|
"color": "0052cc",
|
||||||
|
"description": "Currently being worked.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "status/blocked",
|
||||||
|
"color": "b60205",
|
||||||
|
"description": "Cannot progress without a decision, dependency, credential, or external change.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "status/needs-info",
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "Needs clarifying input before implementation can proceed safely.",
|
||||||
|
"exclusive": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/access",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/addresses",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "GovOPlaN Addresses module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/admin",
|
||||||
|
"color": "006b75",
|
||||||
|
"description": "GovOPlaN Admin module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/appointments",
|
||||||
|
"color": "d876e3",
|
||||||
|
"description": "GovOPlaN Appointments module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/audit",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "GovOPlaN Audit module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/calendar",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "GovOPlaN Calendar module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/campaign",
|
||||||
|
"color": "d876e3",
|
||||||
|
"description": "GovOPlaN campaign module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/cases",
|
||||||
|
"color": "1d76db",
|
||||||
|
"description": "GovOPlaN Cases module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/connectors",
|
||||||
|
"color": "c2e0c6",
|
||||||
|
"description": "GovOPlaN Connectors module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/core",
|
||||||
|
"color": "0052cc",
|
||||||
|
"description": "GovOPlaN core runner, shared primitives, shell, or extension points.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/dms",
|
||||||
|
"color": "c5def5",
|
||||||
|
"description": "GovOPlaN Dms module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/erp",
|
||||||
|
"color": "fef2c0",
|
||||||
|
"description": "GovOPlaN Erp module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/files",
|
||||||
|
"color": "006b75",
|
||||||
|
"description": "GovOPlaN files module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/fit-connect",
|
||||||
|
"color": "fbca04",
|
||||||
|
"description": "GovOPlaN Fit Connect module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/forms",
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "GovOPlaN Forms module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/identity-trust",
|
||||||
|
"color": "d4c5f9",
|
||||||
|
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/identity",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "GovOPlaN Identity module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/idm",
|
||||||
|
"color": "0052cc",
|
||||||
|
"description": "GovOPlaN Idm module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/ledger",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "GovOPlaN Ledger module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/mail",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "GovOPlaN mail module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/notifications",
|
||||||
|
"color": "d876e3",
|
||||||
|
"description": "GovOPlaN Notifications module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/ops",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "GovOPlaN Ops module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/organizations",
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "GovOPlaN Organizations module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/payments",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "GovOPlaN Payments module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/policy",
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "GovOPlaN Policy module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/portal",
|
||||||
|
"color": "1d76db",
|
||||||
|
"description": "GovOPlaN Portal module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/postbox",
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "GovOPlaN Postbox module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/reporting",
|
||||||
|
"color": "c2e0c6",
|
||||||
|
"description": "GovOPlaN Reporting module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/search",
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "GovOPlaN Search module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/scheduling",
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "GovOPlaN Scheduling module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/tasks",
|
||||||
|
"color": "c5def5",
|
||||||
|
"description": "GovOPlaN Tasks module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/templates",
|
||||||
|
"color": "fef2c0",
|
||||||
|
"description": "GovOPlaN Templates module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/tenancy",
|
||||||
|
"color": "e4e669",
|
||||||
|
"description": "GovOPlaN Tenancy module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/web",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "GovOPlaN public website, product page, or publication content.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/workflow",
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "GovOPlaN Workflow module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/xoev",
|
||||||
|
"color": "d4c5f9",
|
||||||
|
"description": "GovOPlaN Xoev module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/xrechnung",
|
||||||
|
"color": "0052cc",
|
||||||
|
"description": "GovOPlaN Xrechnung module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "module/xta-osci",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "GovOPlaN Xta Osci module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/auth",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "Authentication, sessions, access bootstrap, or login behavior.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/tenancy",
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/rbac",
|
||||||
|
"color": "c5def5",
|
||||||
|
"description": "Permissions, roles, delegation, or authorization policy.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/governance",
|
||||||
|
"color": "fef2c0",
|
||||||
|
"description": "Governance policy, audit, privacy, retention, or compliance behavior.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/module-system",
|
||||||
|
"color": "bfe5bf",
|
||||||
|
"description": "Module discovery, manifests, capabilities, routing, or optional integrations.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/migrations",
|
||||||
|
"color": "e4e669",
|
||||||
|
"description": "Alembic migrations, schema bootstrap, or persistence evolution.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/webui",
|
||||||
|
"color": "1d76db",
|
||||||
|
"description": "Shared WebUI shell, frontend components, routing, or frontend tests.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/api",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "HTTP API contracts, routers, schemas, or API smoke behavior.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/db",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "Database sessions, models, transactions, or persistence primitives.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/devex",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "Local developer workflow, scripts, tests, tooling, or release helpers.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/release",
|
||||||
|
"color": "c2e0c6",
|
||||||
|
"description": "Versioning, release locks, tags, packaging, or dependency pins.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/docs",
|
||||||
|
"color": "5319e7",
|
||||||
|
"description": "Durable documentation and project guidance.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "area/marketing",
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "Public website, product messaging, publication copy, or legal page content.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "source/todo-scan",
|
||||||
|
"color": "cccccc",
|
||||||
|
"description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "source/backlog-import",
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "Imported from markdown backlog, roadmap, plan, or TODO files.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "codex/ready",
|
||||||
|
"color": "0e8a16",
|
||||||
|
"description": "Suitable for Codex to pick up with the existing issue context.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "codex/needs-human",
|
||||||
|
"color": "f9d0c4",
|
||||||
|
"description": "Needs an explicit human decision before Codex should implement.",
|
||||||
|
"exclusive": false
|
||||||
|
}
|
||||||
|
]
|
||||||
100
docs/migration-release-baselines.json
Normal file
100
docs/migration-release-baselines.json
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
{
|
||||||
|
"releases": [
|
||||||
|
{
|
||||||
|
"heads": [
|
||||||
|
{
|
||||||
|
"owner": "govoplan-campaign",
|
||||||
|
"revision": "2c3d4e5f7081"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-mail",
|
||||||
|
"revision": "3d4e5f708192"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-core",
|
||||||
|
"revision": "4f2a9c8e7b6d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-identity",
|
||||||
|
"revision": "5c6d7e8f9a10"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-organizations",
|
||||||
|
"revision": "6d7e8f9a0b1c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-idm",
|
||||||
|
"revision": "8f9a0b1c2d3e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-calendar",
|
||||||
|
"revision": "9e0f1a2b3c4d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-files",
|
||||||
|
"revision": "a7b8c9d0e1f3"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"owner_heads": [
|
||||||
|
{
|
||||||
|
"owner": "govoplan-access",
|
||||||
|
"revisions": [
|
||||||
|
"4a5b6c7d8e9f"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-calendar",
|
||||||
|
"revisions": [
|
||||||
|
"9e0f1a2b3c4d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-campaign",
|
||||||
|
"revisions": [
|
||||||
|
"2c3d4e5f7081"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-core",
|
||||||
|
"revisions": [
|
||||||
|
"4f2a9c8e7b6d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-files",
|
||||||
|
"revisions": [
|
||||||
|
"a7b8c9d0e1f3"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-identity",
|
||||||
|
"revisions": [
|
||||||
|
"5c6d7e8f9a10"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-idm",
|
||||||
|
"revisions": [
|
||||||
|
"8f9a0b1c2d3e"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-mail",
|
||||||
|
"revisions": [
|
||||||
|
"3d4e5f708192"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-organizations",
|
||||||
|
"revisions": [
|
||||||
|
"6d7e8f9a0b1c"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"recorded_at": "2026-07-11T00:19:07Z",
|
||||||
|
"release": "0.1.7",
|
||||||
|
"squash_policy": "reviewed-manual"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
151
docs/module-package-catalog.example.json
Normal file
151
docs/module-package-catalog.example.json
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
{
|
||||||
|
"catalog_version": "1",
|
||||||
|
"channel": "stable",
|
||||||
|
"sequence": 14,
|
||||||
|
"generated_at": "2026-07-07T00:00:00Z",
|
||||||
|
"expires_at": "2030-12-31T23:59:59Z",
|
||||||
|
"modules": [
|
||||||
|
{
|
||||||
|
"module_id": "files",
|
||||||
|
"name": "Files",
|
||||||
|
"description": "Managed file spaces and campaign attachment integration.",
|
||||||
|
"version": "0.1.4",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-files",
|
||||||
|
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||||
|
"webui_package": "@govoplan/files-webui",
|
||||||
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||||
|
"migration_safety": "forward_only",
|
||||||
|
"migration_notes": "Database rollback requires restoring the pre-update snapshot.",
|
||||||
|
"migration_after": ["access"],
|
||||||
|
"migration_before": ["campaigns"],
|
||||||
|
"migration_tasks": [
|
||||||
|
{
|
||||||
|
"task_id": "backfill_spaces",
|
||||||
|
"phase": "post_migration_backfill",
|
||||||
|
"summary": "Backfill file spaces after the schema migration.",
|
||||||
|
"task_version": "1",
|
||||||
|
"safety": "forward_only",
|
||||||
|
"idempotent": true,
|
||||||
|
"timeout_seconds": 300
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"current_version_min": "0.1.0",
|
||||||
|
"current_version_max_exclusive": "0.2.0",
|
||||||
|
"bridge_release": true,
|
||||||
|
"bridge_notes": "Keeps the 0.1.x campaign attachment interface while introducing the 0.2.x contract.",
|
||||||
|
"allow_downgrade": false,
|
||||||
|
"allow_same_version": false,
|
||||||
|
"recovery_tested": true,
|
||||||
|
"recovery_notes": "Snapshot restore and forward recovery were rehearsed on the release candidate dataset.",
|
||||||
|
"provides_interfaces": [
|
||||||
|
{
|
||||||
|
"name": "files.campaign_attachments",
|
||||||
|
"version": "1.4.0"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"requires_interfaces": [
|
||||||
|
{
|
||||||
|
"name": "campaigns.access",
|
||||||
|
"version_min": "1.0.0",
|
||||||
|
"version_max_exclusive": "2.0.0",
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-webui-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-webui-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"license_features": ["module.files"],
|
||||||
|
"tags": ["official", "service-module"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module_id": "mail",
|
||||||
|
"name": "Mail",
|
||||||
|
"description": "SMTP/IMAP profile management and read-only mailbox access.",
|
||||||
|
"version": "0.1.4",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-mail",
|
||||||
|
"python_ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4",
|
||||||
|
"webui_package": "@govoplan/mail-webui",
|
||||||
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4",
|
||||||
|
"provides_interfaces": [
|
||||||
|
{
|
||||||
|
"name": "mail.campaign_delivery",
|
||||||
|
"version": "1.4.0"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-webui-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-webui-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"license_features": ["module.mail"],
|
||||||
|
"tags": ["official", "service-module"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module_id": "dashboard",
|
||||||
|
"name": "Dashboard",
|
||||||
|
"description": "Configurable user home assembled from module-provided widgets.",
|
||||||
|
"version": "0.1.6",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-dashboard",
|
||||||
|
"python_ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6",
|
||||||
|
"webui_package": "@govoplan/dashboard-webui",
|
||||||
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6",
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-0.1.6.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-0.1.6.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.6"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-webui-0.1.6.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-webui-0.1.6.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"license_features": ["module.dashboard"],
|
||||||
|
"tags": ["official", "platform-module"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"signatures": [
|
||||||
|
{
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": "release-key-1",
|
||||||
|
"value": "<base64 signature over this JSON object without signature or signatures fields>"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": "release-key-2",
|
||||||
|
"value": "<optional second signature for key rotation>"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-core"
|
name = "govoplan-core"
|
||||||
version = "0.1.3"
|
version = "0.1.7"
|
||||||
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
@@ -12,10 +12,10 @@ license = { file = "LICENSE" }
|
|||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"SQLAlchemy>=2.0,<3",
|
"SQLAlchemy>=2.0,<3",
|
||||||
"fastapi>=0.115,<1",
|
"fastapi>=0.139,<1",
|
||||||
"pydantic>=2,<3",
|
"pydantic>=2,<3",
|
||||||
"pydantic-settings>=2,<3",
|
"pydantic-settings>=2,<3",
|
||||||
"cryptography>=44,<45",
|
"cryptography>=48.0.1,<50",
|
||||||
"celery>=5,<6",
|
"celery>=5,<6",
|
||||||
"redis>=5,<6",
|
"redis>=5,<6",
|
||||||
"alembic>=1,<2",
|
"alembic>=1,<2",
|
||||||
@@ -28,15 +28,17 @@ where = ["src"]
|
|||||||
govoplan_core = ["py.typed"]
|
govoplan_core = ["py.typed"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
govoplan-config = "govoplan_core.commands.config:main"
|
||||||
govoplan-devserver = "govoplan_core.devserver:main"
|
govoplan-devserver = "govoplan_core.devserver:main"
|
||||||
|
govoplan-module-install-plan = "govoplan_core.commands.module_install_plan:main"
|
||||||
[project.entry-points."govoplan.modules"]
|
govoplan-module-installer = "govoplan_core.commands.module_installer:main"
|
||||||
access = "govoplan_core.access.manifest:get_manifest"
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
server = [
|
server = [
|
||||||
|
"psycopg[binary]>=3.2,<4",
|
||||||
"uvicorn[standard]>=0.32,<1",
|
"uvicorn[standard]>=0.32,<1",
|
||||||
]
|
]
|
||||||
dev = [
|
dev = [
|
||||||
"httpx==0.28.1",
|
"httpx==0.28.1",
|
||||||
|
"httpx2>=2.5,<3",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,5 +1,18 @@
|
|||||||
-e .[server]
|
-e .[server]
|
||||||
|
-e ../govoplan-tenancy
|
||||||
|
-e ../govoplan-organizations
|
||||||
|
-e ../govoplan-identity
|
||||||
|
-e ../govoplan-access
|
||||||
|
-e ../govoplan-admin
|
||||||
|
-e ../govoplan-policy
|
||||||
|
-e ../govoplan-audit
|
||||||
|
-e ../govoplan-dashboard
|
||||||
-e ../govoplan-files
|
-e ../govoplan-files
|
||||||
-e ../govoplan-mail
|
-e ../govoplan-mail
|
||||||
-e ../govoplan-campaign
|
-e ../govoplan-campaign
|
||||||
|
-e ../govoplan-calendar
|
||||||
|
-e ../govoplan-docs
|
||||||
|
-e ../govoplan-ops
|
||||||
httpx==0.28.1
|
httpx==0.28.1
|
||||||
|
httpx2>=2.5,<3
|
||||||
|
pip-audit>=2.9,<3
|
||||||
|
|||||||
@@ -1,6 +1,18 @@
|
|||||||
# Release install from tagged module repositories.
|
# Release install from tagged module repositories.
|
||||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
||||||
.[server]
|
.[server]
|
||||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.1
|
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
|
||||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.1
|
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
|
||||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.1
|
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
|
||||||
|
govoplan-idm @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-idm.git@v0.1.6
|
||||||
|
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
|
||||||
|
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
|
||||||
|
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
|
||||||
|
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
|
||||||
|
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
|
||||||
|
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
|
||||||
|
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
|
||||||
|
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
|
||||||
|
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
|
||||||
|
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
|
||||||
|
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6
|
||||||
|
|||||||
37
scripts/check-dependency-audits.sh
Normal file
37
scripts/check-dependency-audits.sh
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||||
|
NODE_BIN="$(dirname "$NPM")"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
if [[ ! -x "$NPM" ]]; then
|
||||||
|
NPM=npm
|
||||||
|
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
||||||
|
|
||||||
|
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
||||||
|
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
||||||
|
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
||||||
|
exit 127
|
||||||
|
fi
|
||||||
|
|
||||||
|
python_status=0
|
||||||
|
npm_status=0
|
||||||
|
|
||||||
|
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
||||||
|
|
||||||
|
cd "$ROOT/webui"
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
||||||
|
|
||||||
|
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
||||||
|
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
158
scripts/check-dependency-hygiene.sh
Normal file
158
scripts/check-dependency-hygiene.sh
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if importlib.util.find_spec("pip") is None:
|
||||||
|
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
||||||
|
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
||||||
|
print(" python -m venv .venv", file=sys.stderr)
|
||||||
|
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
||||||
|
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
||||||
|
raise SystemExit(127)
|
||||||
|
PY
|
||||||
|
|
||||||
|
"$PYTHON" -m pip check
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.metadata
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
prefix = pathlib.Path(sys.prefix)
|
||||||
|
legacy_patterns = (
|
||||||
|
"__editable__.govoplan_module_multimailer-*.pth",
|
||||||
|
"__editable___govoplan_module_multimailer_*_finder.py",
|
||||||
|
"govoplan_module_multimailer-*.dist-info",
|
||||||
|
)
|
||||||
|
problems: list[pathlib.Path] = []
|
||||||
|
|
||||||
|
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
||||||
|
for pattern in legacy_patterns:
|
||||||
|
problems.extend(site_packages.glob(pattern))
|
||||||
|
|
||||||
|
for dist in importlib.metadata.distributions():
|
||||||
|
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
||||||
|
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
||||||
|
|
||||||
|
if problems:
|
||||||
|
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
||||||
|
for path in sorted(set(problems)):
|
||||||
|
print(f" {path}", file=sys.stderr)
|
||||||
|
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
print("No stale legacy multimailer package metadata found.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path.cwd()
|
||||||
|
scan_roots = [
|
||||||
|
root / "src",
|
||||||
|
root / "tests",
|
||||||
|
root.parent / "govoplan-access" / "src",
|
||||||
|
root.parent / "govoplan-admin" / "src",
|
||||||
|
root.parent / "govoplan-audit" / "src",
|
||||||
|
root.parent / "govoplan-calendar" / "src",
|
||||||
|
root.parent / "govoplan-campaign" / "src",
|
||||||
|
root.parent / "govoplan-dashboard" / "src",
|
||||||
|
root.parent / "govoplan-files" / "src",
|
||||||
|
root.parent / "govoplan-identity" / "src",
|
||||||
|
root.parent / "govoplan-mail" / "src",
|
||||||
|
root.parent / "govoplan-ops" / "src",
|
||||||
|
root.parent / "govoplan-organizations" / "src",
|
||||||
|
root.parent / "govoplan-policy" / "src",
|
||||||
|
root.parent / "govoplan-tenancy" / "src",
|
||||||
|
]
|
||||||
|
deprecated_tokens = {
|
||||||
|
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
||||||
|
}
|
||||||
|
|
||||||
|
hits: list[str] = []
|
||||||
|
for scan_root in scan_roots:
|
||||||
|
if not scan_root.exists():
|
||||||
|
continue
|
||||||
|
for path in scan_root.rglob("*.py"):
|
||||||
|
try:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
continue
|
||||||
|
for token, replacement in deprecated_tokens.items():
|
||||||
|
if token in text:
|
||||||
|
hits.append(f"{path}: deprecated {token}. {replacement}")
|
||||||
|
|
||||||
|
if hits:
|
||||||
|
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
||||||
|
print("\n".join(hits), file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
print("Deprecated framework constant scan passed.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
||||||
|
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
||||||
|
0|false|False|no|No)
|
||||||
|
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
||||||
|
;;
|
||||||
|
auto)
|
||||||
|
if "$PYTHON" - <<'PY'
|
||||||
|
import importlib.util
|
||||||
|
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||||
|
else
|
||||||
|
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
from starlette.exceptions import StarletteDeprecationWarning
|
||||||
|
|
||||||
|
warnings.simplefilter("error", StarletteDeprecationWarning)
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
|
||||||
|
@app.get("/dependency-hygiene-ping")
|
||||||
|
def ping() -> dict[str, bool]:
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
response = client.get("/dependency-hygiene-ping")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == {"ok": True}
|
||||||
|
|
||||||
|
print("TestClient deprecation smoke passed.")
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Dependency hygiene check passed."
|
||||||
@@ -5,11 +5,19 @@ ROOT="/mnt/DATA/git/govoplan-core"
|
|||||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
||||||
NPM="$NODE/npm"
|
NPM="$NODE/npm"
|
||||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||||
|
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||||
|
|
||||||
|
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
||||||
|
|
||||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
||||||
|
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||||
|
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
||||||
|
unset npm_config_tmp NPM_CONFIG_TMP
|
||||||
|
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||||
|
|
||||||
./.venv/bin/python - <<'PY'
|
./.venv/bin/python - <<'PY'
|
||||||
import ast
|
import ast
|
||||||
import pathlib
|
import pathlib
|
||||||
@@ -17,6 +25,13 @@ import sys
|
|||||||
|
|
||||||
roots = [
|
roots = [
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
||||||
@@ -42,7 +57,8 @@ if errors:
|
|||||||
print(f"AST syntax check passed for {count} Python files")
|
print(f"AST syntax check passed for {count} Python files")
|
||||||
PY
|
PY
|
||||||
|
|
||||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||||
|
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
./.venv/bin/python -m unittest tests.test_module_system
|
||||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||||
|
|||||||
23
scripts/check-module-matrix.sh
Normal file
23
scripts/check-module-matrix.sh
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||||
|
NODE_BIN="$(dirname "$NPM")"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
if [[ ! -x "$NPM" ]]; then
|
||||||
|
NPM=npm
|
||||||
|
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
||||||
|
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||||
|
|
||||||
|
cd "$ROOT/webui"
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
||||||
278
scripts/check-release-integration.sh
Normal file
278
scripts/check-release-integration.sh
Normal file
@@ -0,0 +1,278 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||||
|
NODE_BIN="$(dirname "$NPM")"
|
||||||
|
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||||
|
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||||
|
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-integration.XXXXXXXX")"
|
||||||
|
|
||||||
|
trap 'rm -rf "$WORK_ROOT"; rm -f "$NPM_USERCONFIG"' EXIT
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
if [[ ! -x "$NPM" ]]; then
|
||||||
|
NPM=npm
|
||||||
|
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
RELEASE_VERSION="$("$PYTHON" - <<'PY'
|
||||||
|
from pathlib import Path
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
with (Path.cwd() / "pyproject.toml").open("rb") as handle:
|
||||||
|
print(tomllib.load(handle)["project"]["version"])
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
RELEASE_TAG="${GOVOPLAN_RELEASE_TAG:-v$RELEASE_VERSION}"
|
||||||
|
|
||||||
|
export RELEASE_TAG RELEASE_VERSION
|
||||||
|
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
|
||||||
|
export PATH="$WEBUI_BIN:$NODE_BIN:$PATH"
|
||||||
|
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||||
|
unset npm_config_tmp NPM_CONFIG_TMP
|
||||||
|
|
||||||
|
run_step() {
|
||||||
|
echo
|
||||||
|
echo "==> $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
retry() {
|
||||||
|
local attempt status
|
||||||
|
for attempt in 1 2 3; do
|
||||||
|
if "$@"; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
status=$?
|
||||||
|
if [[ "$attempt" == 3 ]]; then
|
||||||
|
return "$status"
|
||||||
|
fi
|
||||||
|
sleep $((attempt * 10))
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
run_discovered_tests() {
|
||||||
|
local suite_dir="$1"
|
||||||
|
local status
|
||||||
|
if ! find "$suite_dir" -type f -name 'test*.py' -print -quit | grep -q .; then
|
||||||
|
echo "No unittest test files found under $suite_dir; skipping."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
set +e
|
||||||
|
"$PYTHON" -m unittest discover -s "$suite_dir"
|
||||||
|
status=$?
|
||||||
|
set -e
|
||||||
|
if [[ "$status" == 5 ]]; then
|
||||||
|
echo "No unittest tests discovered under $suite_dir; skipping."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
install_cloned_webui_dependencies() {
|
||||||
|
local webui_dir="$1"
|
||||||
|
if [[ ! -f "$webui_dir/package.json" ]]; then
|
||||||
|
echo "No package.json found under $webui_dir; skipping WebUI dependency install."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
(
|
||||||
|
cd "$webui_dir"
|
||||||
|
retry "$NPM" install --prefer-online --include=dev --no-save
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
run_step "Validate release refs and installed package metadata"
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.metadata as metadata
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
root = Path.cwd()
|
||||||
|
release_requirements = root / "requirements-release.txt"
|
||||||
|
release_webui = root / "webui" / "package.release.json"
|
||||||
|
expected_version = os.environ["RELEASE_VERSION"]
|
||||||
|
expected_tag = os.environ["RELEASE_TAG"]
|
||||||
|
errors: list[str] = []
|
||||||
|
|
||||||
|
python_requirements: list[tuple[str, str, str]] = []
|
||||||
|
for line in release_requirements.read_text(encoding="utf-8").splitlines():
|
||||||
|
match = re.match(r"^(govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://(.+?\.git)@(v[0-9]+\.[0-9]+\.[0-9]+)$", line.strip())
|
||||||
|
if match:
|
||||||
|
python_requirements.append((match.group(1), f"ssh://{match.group(2)}", match.group(3)))
|
||||||
|
|
||||||
|
if not python_requirements:
|
||||||
|
errors.append("No GovOPlaN git+ssh release requirements found.")
|
||||||
|
|
||||||
|
for package_name, repo_url, tag in python_requirements:
|
||||||
|
if tag != expected_tag:
|
||||||
|
errors.append(f"{package_name} release requirement uses {tag!r}, expected {expected_tag!r}.")
|
||||||
|
version = metadata.version(package_name)
|
||||||
|
if version != tag.removeprefix("v"):
|
||||||
|
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "ls-remote", "--tags", repo_url, f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"],
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
timeout=45,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0 or f"refs/tags/{tag}" not in result.stdout:
|
||||||
|
errors.append(f"{package_name} tag {tag} is not readable from {repo_url}: {result.stderr.strip()}")
|
||||||
|
|
||||||
|
release_package = json.loads(release_webui.read_text(encoding="utf-8"))
|
||||||
|
if release_package.get("version") != expected_version:
|
||||||
|
errors.append(f"Core release WebUI version is {release_package.get('version')!r}, expected {expected_version!r}.")
|
||||||
|
|
||||||
|
webui_dependencies = release_package.get("dependencies", {})
|
||||||
|
for package_name, spec in sorted(webui_dependencies.items()):
|
||||||
|
if not package_name.startswith("@govoplan/"):
|
||||||
|
continue
|
||||||
|
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
|
||||||
|
if tag != expected_tag:
|
||||||
|
errors.append(f"{package_name} release dependency uses {tag!r}, expected {expected_tag}.")
|
||||||
|
package_json = root / "webui" / "node_modules" / package_name / "package.json"
|
||||||
|
if not package_json.exists():
|
||||||
|
errors.append(f"{package_name} is missing from release node_modules.")
|
||||||
|
continue
|
||||||
|
installed = json.loads(package_json.read_text(encoding="utf-8"))
|
||||||
|
if installed.get("version") != expected_version:
|
||||||
|
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {expected_version!r}.")
|
||||||
|
|
||||||
|
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
|
||||||
|
root_peers = release_package.get("peerDependencies", {})
|
||||||
|
for package_json in sorted((root / "webui" / "node_modules" / "@govoplan").glob("*/package.json")):
|
||||||
|
package = json.loads(package_json.read_text(encoding="utf-8"))
|
||||||
|
peers = package.get("peerDependencies", {})
|
||||||
|
for name in shared_peers:
|
||||||
|
if name in peers and name in root_peers and peers[name] != root_peers[name]:
|
||||||
|
errors.append(f"{package.get('name')} peer {name}={peers[name]!r}, expected {root_peers[name]!r}.")
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
print("\n".join(errors), file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
print(f"Release refs and metadata passed for {len(python_requirements)} Python packages and {len(webui_dependencies)} WebUI packages.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
run_step "Validate installed module manifests and registry"
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
"access",
|
||||||
|
"admin",
|
||||||
|
"audit",
|
||||||
|
"calendar",
|
||||||
|
"campaigns",
|
||||||
|
"dashboard",
|
||||||
|
"docs",
|
||||||
|
"files",
|
||||||
|
"identity",
|
||||||
|
"idm",
|
||||||
|
"mail",
|
||||||
|
"ops",
|
||||||
|
"organizations",
|
||||||
|
"policy",
|
||||||
|
"tenancy",
|
||||||
|
}
|
||||||
|
|
||||||
|
manifests = available_module_manifests()
|
||||||
|
missing = sorted(expected - set(manifests))
|
||||||
|
if missing:
|
||||||
|
print(f"Missing installed module manifests: {', '.join(missing)}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
registry = build_platform_registry(tuple(sorted(expected)))
|
||||||
|
snapshot = registry.validate()
|
||||||
|
print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests))
|
||||||
|
PY
|
||||||
|
|
||||||
|
run_step "Run SQLite migration smoke for release modules"
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from govoplan_core.db.migrations import migrate_database
|
||||||
|
|
||||||
|
enabled_modules = (
|
||||||
|
"tenancy",
|
||||||
|
"organizations",
|
||||||
|
"identity",
|
||||||
|
"idm",
|
||||||
|
"access",
|
||||||
|
"admin",
|
||||||
|
"dashboard",
|
||||||
|
"policy",
|
||||||
|
"audit",
|
||||||
|
"campaigns",
|
||||||
|
"files",
|
||||||
|
"mail",
|
||||||
|
"calendar",
|
||||||
|
"docs",
|
||||||
|
"ops",
|
||||||
|
)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="govoplan-release-migration-") as tmp:
|
||||||
|
database_url = f"sqlite:///{Path(tmp) / 'release-integration.sqlite3'}"
|
||||||
|
result = migrate_database(database_url=database_url, enabled_modules=enabled_modules)
|
||||||
|
if not result.current_revision:
|
||||||
|
raise SystemExit("Migration smoke did not produce an Alembic revision.")
|
||||||
|
print(f"SQLite release migration smoke reached {result.current_revision}.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
run_step "Clone release module test sources"
|
||||||
|
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
|
||||||
|
git clone --depth 1 --branch "$RELEASE_TAG" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
||||||
|
done
|
||||||
|
|
||||||
|
run_step "Run core backend release tests"
|
||||||
|
"$PYTHON" -m unittest \
|
||||||
|
tests.test_module_system \
|
||||||
|
tests.test_access_contracts \
|
||||||
|
tests.test_identity_organization_contracts \
|
||||||
|
tests.test_core_events \
|
||||||
|
tests.test_policy_contracts
|
||||||
|
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||||
|
|
||||||
|
run_step "Run cloned module backend tests"
|
||||||
|
run_discovered_tests "$WORK_ROOT/govoplan-mail/tests"
|
||||||
|
run_discovered_tests "$WORK_ROOT/govoplan-calendar/tests"
|
||||||
|
run_discovered_tests "$WORK_ROOT/govoplan-campaign/tests"
|
||||||
|
|
||||||
|
run_step "Run core WebUI release tests and build"
|
||||||
|
cd "$ROOT/webui"
|
||||||
|
"$NPM" run test:mail-components
|
||||||
|
"$NPM" run test:module-capabilities
|
||||||
|
"$NPM" run test:module-permutations
|
||||||
|
"$NPM" run build
|
||||||
|
|
||||||
|
run_step "Run cloned module WebUI tests"
|
||||||
|
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-mail/webui"
|
||||||
|
cd "$WORK_ROOT/govoplan-mail/webui"
|
||||||
|
"$NPM" run test:mail-ui
|
||||||
|
|
||||||
|
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-campaign/webui"
|
||||||
|
cd "$WORK_ROOT/govoplan-campaign/webui"
|
||||||
|
"$NPM" run test:policy-ui
|
||||||
|
"$NPM" run test:template-preview
|
||||||
|
"$NPM" run test:import-utils
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Release integration check passed."
|
||||||
236
scripts/check_dependency_boundaries.py
Normal file
236
scripts/check_dependency_boundaries.py
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
REPOS = {
|
||||||
|
"core": ROOT / "src" / "govoplan_core",
|
||||||
|
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||||
|
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||||
|
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||||
|
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
||||||
|
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
||||||
|
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||||
|
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||||
|
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
||||||
|
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||||
|
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||||
|
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||||
|
}
|
||||||
|
PREFIXES = {
|
||||||
|
"core": "govoplan_core",
|
||||||
|
"access": "govoplan_access",
|
||||||
|
"admin": "govoplan_admin",
|
||||||
|
"tenancy": "govoplan_tenancy",
|
||||||
|
"organizations": "govoplan_organizations",
|
||||||
|
"identity": "govoplan_identity",
|
||||||
|
"policy": "govoplan_policy",
|
||||||
|
"audit": "govoplan_audit",
|
||||||
|
"dashboard": "govoplan_dashboard",
|
||||||
|
"files": "govoplan_files",
|
||||||
|
"mail": "govoplan_mail",
|
||||||
|
"campaign": "govoplan_campaign",
|
||||||
|
}
|
||||||
|
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||||
|
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
||||||
|
WEBUI_REPOS = {
|
||||||
|
"core": ROOT / "webui",
|
||||||
|
"files": ROOT.parent / "govoplan-files" / "webui",
|
||||||
|
"mail": ROOT.parent / "govoplan-mail" / "webui",
|
||||||
|
"campaign": ROOT.parent / "govoplan-campaign" / "webui",
|
||||||
|
}
|
||||||
|
WEBUI_PACKAGES = {
|
||||||
|
"core": "@govoplan/core-webui",
|
||||||
|
"files": "@govoplan/files-webui",
|
||||||
|
"mail": "@govoplan/mail-webui",
|
||||||
|
"campaign": "@govoplan/campaign-webui",
|
||||||
|
}
|
||||||
|
WEBUI_IMPORT_RE = re.compile(
|
||||||
|
r"(?:from\s+|import\s*\(\s*|require\s*\(\s*|export\s+[^;]*?\s+from\s+)"
|
||||||
|
r"[\"'](?P<package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||||
|
r"|import\s+[\"'](?P<side_effect_package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AllowlistedImport:
|
||||||
|
owner: str
|
||||||
|
relative_path: str
|
||||||
|
imported_owner: str
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
|
||||||
|
# Transitional exceptions. This should remain empty; add entries only with a
|
||||||
|
# dated removal plan and a capability/API contract issue.
|
||||||
|
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
|
||||||
|
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Violation:
|
||||||
|
kind: str
|
||||||
|
owner: str
|
||||||
|
path: Path
|
||||||
|
lineno: int
|
||||||
|
imported: str
|
||||||
|
imported_owner: str
|
||||||
|
|
||||||
|
|
||||||
|
def _import_owner(module_name: str) -> str | None:
|
||||||
|
for owner, prefix in PREFIXES.items():
|
||||||
|
if module_name == prefix or module_name.startswith(f"{prefix}."):
|
||||||
|
return owner
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _imports(path: Path) -> list[tuple[int, str]]:
|
||||||
|
tree = ast.parse(path.read_text(), filename=str(path))
|
||||||
|
result: list[tuple[int, str]] = []
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if isinstance(node, ast.Import):
|
||||||
|
for alias in node.names:
|
||||||
|
result.append((node.lineno, alias.name))
|
||||||
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||||
|
result.append((node.lineno, node.module))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _relative(owner: str, path: Path) -> str:
|
||||||
|
return path.relative_to(REPOS[owner]).as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
|
||||||
|
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
|
||||||
|
|
||||||
|
|
||||||
|
def _violations() -> list[Violation]:
|
||||||
|
violations: list[Violation] = []
|
||||||
|
for owner, root in REPOS.items():
|
||||||
|
if not root.exists():
|
||||||
|
continue
|
||||||
|
for path in root.rglob("*.py"):
|
||||||
|
if "__pycache__" in path.parts:
|
||||||
|
continue
|
||||||
|
for lineno, imported in _imports(path):
|
||||||
|
imported_owner = _import_owner(imported)
|
||||||
|
if imported_owner is None or imported_owner == owner:
|
||||||
|
continue
|
||||||
|
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||||
|
if not _allowed(owner, path, imported_owner):
|
||||||
|
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||||
|
elif owner == "access" and imported_owner in FEATURE_OWNERS:
|
||||||
|
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||||
|
elif owner in PLATFORM_OWNERS and imported_owner == "access":
|
||||||
|
if not _allowed(owner, path, imported_owner):
|
||||||
|
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||||
|
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||||
|
if not _allowed(owner, path, imported_owner):
|
||||||
|
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||||
|
elif owner in FEATURE_OWNERS and imported_owner == "access":
|
||||||
|
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||||
|
violations.extend(_webui_violations())
|
||||||
|
return violations
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_package_owner(package_name: str) -> str | None:
|
||||||
|
for owner, package in WEBUI_PACKAGES.items():
|
||||||
|
if package_name == package:
|
||||||
|
return owner
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_source_files(root: Path) -> list[Path]:
|
||||||
|
source_root = root / "src"
|
||||||
|
if not source_root.exists():
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
path
|
||||||
|
for path in source_root.rglob("*")
|
||||||
|
if path.suffix in {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
|
||||||
|
and "node_modules" not in path.parts
|
||||||
|
and "dist" not in path.parts
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_dependency_fields(payload: object) -> dict[str, object]:
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return {}
|
||||||
|
result: dict[str, object] = {}
|
||||||
|
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
||||||
|
value = payload.get(key)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
result.update(value)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_package_json_violations(owner: str, root: Path) -> list[Violation]:
|
||||||
|
violations: list[Violation] = []
|
||||||
|
package_paths = [root / "package.json"]
|
||||||
|
if root.name == "webui":
|
||||||
|
package_paths.append(root.parent / "package.json")
|
||||||
|
for package_path in tuple(dict.fromkeys(package_paths)):
|
||||||
|
if not package_path.exists():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
payload = json.loads(package_path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
return [Violation("webui-package", owner, package_path, exc.lineno, "invalid package.json", owner)]
|
||||||
|
for package_name in _webui_dependency_fields(payload):
|
||||||
|
imported_owner = _webui_package_owner(str(package_name))
|
||||||
|
if imported_owner is None or imported_owner == owner:
|
||||||
|
continue
|
||||||
|
if owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||||
|
violations.append(Violation("webui-package", owner, package_path, 1, str(package_name), imported_owner))
|
||||||
|
return violations
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_source_violations(owner: str, root: Path) -> list[Violation]:
|
||||||
|
violations: list[Violation] = []
|
||||||
|
for path in _webui_source_files(root):
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
for match in WEBUI_IMPORT_RE.finditer(text):
|
||||||
|
package_name = match.group("package") or match.group("side_effect_package")
|
||||||
|
imported_owner = _webui_package_owner(package_name)
|
||||||
|
if imported_owner is None or imported_owner == owner:
|
||||||
|
continue
|
||||||
|
lineno = text.count("\n", 0, match.start()) + 1
|
||||||
|
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||||
|
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||||
|
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||||
|
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||||
|
return violations
|
||||||
|
|
||||||
|
|
||||||
|
def _webui_violations() -> list[Violation]:
|
||||||
|
violations: list[Violation] = []
|
||||||
|
for owner, root in WEBUI_REPOS.items():
|
||||||
|
if not root.exists():
|
||||||
|
continue
|
||||||
|
violations.extend(_webui_package_json_violations(owner, root))
|
||||||
|
violations.extend(_webui_source_violations(owner, root))
|
||||||
|
return violations
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
violations = _violations()
|
||||||
|
if not violations:
|
||||||
|
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
print("Dependency boundary violations:")
|
||||||
|
for item in violations:
|
||||||
|
print(
|
||||||
|
f"- {item.path}:{item.lineno}: {item.owner} {item.kind} imports {item.imported} "
|
||||||
|
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
|
||||||
|
)
|
||||||
|
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
83
scripts/generate-catalog-keypair.py
Normal file
83
scripts/generate-catalog-keypair.py
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import stat
|
||||||
|
|
||||||
|
from cryptography.hazmat.primitives import serialization
|
||||||
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
|
||||||
|
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
|
||||||
|
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
|
||||||
|
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
|
||||||
|
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
|
||||||
|
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
private_path = args.private_key.expanduser()
|
||||||
|
public_path = args.public_key.expanduser() if args.public_key else None
|
||||||
|
keyring_path = args.keyring.expanduser() if args.keyring else None
|
||||||
|
if private_path.exists() and not args.force:
|
||||||
|
raise SystemExit(f"Private key already exists: {private_path}")
|
||||||
|
if public_path is not None and public_path.exists() and not args.force:
|
||||||
|
raise SystemExit(f"Public key already exists: {public_path}")
|
||||||
|
|
||||||
|
private_key = Ed25519PrivateKey.generate()
|
||||||
|
private_bytes = private_key.private_bytes(
|
||||||
|
encoding=serialization.Encoding.PEM,
|
||||||
|
format=serialization.PrivateFormat.PKCS8,
|
||||||
|
encryption_algorithm=serialization.NoEncryption(),
|
||||||
|
)
|
||||||
|
public_bytes = private_key.public_key().public_bytes(
|
||||||
|
encoding=serialization.Encoding.Raw,
|
||||||
|
format=serialization.PublicFormat.Raw,
|
||||||
|
)
|
||||||
|
public_base64 = base64.b64encode(public_bytes).decode("ascii")
|
||||||
|
|
||||||
|
private_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
private_path.write_bytes(private_bytes)
|
||||||
|
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||||
|
|
||||||
|
if public_path is not None:
|
||||||
|
public_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
public_path.write_text(public_base64 + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
if keyring_path is not None:
|
||||||
|
keyring_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
keyring = {
|
||||||
|
"keyring_version": "1",
|
||||||
|
"purpose": "govoplan module package catalog signatures",
|
||||||
|
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
|
||||||
|
"keys": [
|
||||||
|
{
|
||||||
|
"key_id": args.key_id,
|
||||||
|
"status": args.status,
|
||||||
|
"public_key": public_base64,
|
||||||
|
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
print(f"private_key={private_path}")
|
||||||
|
if public_path is not None:
|
||||||
|
print(f"public_key={public_path}")
|
||||||
|
if keyring_path is not None:
|
||||||
|
print(f"keyring={keyring_path}")
|
||||||
|
print(f"key_id={args.key_id}")
|
||||||
|
print(f"public_key_base64={public_base64}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
405
scripts/generate-release-catalog.py
Normal file
405
scripts/generate-release-catalog.py
Normal file
@@ -0,0 +1,405 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate and sign a GovOPlaN module package release catalog."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from cryptography.hazmat.primitives import serialization
|
||||||
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||||
|
|
||||||
|
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||||
|
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CatalogModule:
|
||||||
|
module_id: str
|
||||||
|
repo: str
|
||||||
|
python_package: str
|
||||||
|
name: str
|
||||||
|
description: str
|
||||||
|
tags: tuple[str, ...]
|
||||||
|
webui_package: str | None = None
|
||||||
|
provides_interfaces: tuple[dict[str, object], ...] = ()
|
||||||
|
requires_interfaces: tuple[dict[str, object], ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
CATALOG_MODULES = (
|
||||||
|
CatalogModule(
|
||||||
|
module_id="tenancy",
|
||||||
|
repo="govoplan-tenancy",
|
||||||
|
python_package="govoplan-tenancy",
|
||||||
|
name="Tenancy",
|
||||||
|
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="organizations",
|
||||||
|
repo="govoplan-organizations",
|
||||||
|
python_package="govoplan-organizations",
|
||||||
|
name="Organizations",
|
||||||
|
description="Organization units, functions, and account-held function assignments.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="identity",
|
||||||
|
repo="govoplan-identity",
|
||||||
|
python_package="govoplan-identity",
|
||||||
|
name="Identity",
|
||||||
|
description="Canonical identities and links between identities and platform accounts.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="access",
|
||||||
|
repo="govoplan-access",
|
||||||
|
python_package="govoplan-access",
|
||||||
|
name="Access",
|
||||||
|
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/access-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="admin",
|
||||||
|
repo="govoplan-admin",
|
||||||
|
python_package="govoplan-admin",
|
||||||
|
name="Admin",
|
||||||
|
description="System settings, governance templates, module management, and admin shell contributions.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/admin-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="policy",
|
||||||
|
repo="govoplan-policy",
|
||||||
|
python_package="govoplan-policy",
|
||||||
|
name="Policy",
|
||||||
|
description="Policy and governance capability module.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/policy-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="audit",
|
||||||
|
repo="govoplan-audit",
|
||||||
|
python_package="govoplan-audit",
|
||||||
|
name="Audit",
|
||||||
|
description="Audit-log storage and audit administration routes.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/audit-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="dashboard",
|
||||||
|
repo="govoplan-dashboard",
|
||||||
|
python_package="govoplan-dashboard",
|
||||||
|
name="Dashboard",
|
||||||
|
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/dashboard-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="files",
|
||||||
|
repo="govoplan-files",
|
||||||
|
python_package="govoplan-files",
|
||||||
|
name="Files",
|
||||||
|
description="Managed file spaces and campaign attachment integration.",
|
||||||
|
tags=("official", "service-module"),
|
||||||
|
webui_package="@govoplan/files-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="mail",
|
||||||
|
repo="govoplan-mail",
|
||||||
|
python_package="govoplan-mail",
|
||||||
|
name="Mail",
|
||||||
|
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
||||||
|
tags=("official", "service-module"),
|
||||||
|
webui_package="@govoplan/mail-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="campaigns",
|
||||||
|
repo="govoplan-campaign",
|
||||||
|
python_package="govoplan-campaign",
|
||||||
|
name="Campaigns",
|
||||||
|
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
||||||
|
tags=("official", "business-module"),
|
||||||
|
webui_package="@govoplan/campaign-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="calendar",
|
||||||
|
repo="govoplan-calendar",
|
||||||
|
python_package="govoplan-calendar",
|
||||||
|
name="Calendar",
|
||||||
|
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
||||||
|
tags=("official", "service-module"),
|
||||||
|
webui_package="@govoplan/calendar-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="docs",
|
||||||
|
repo="govoplan-docs",
|
||||||
|
python_package="govoplan-docs",
|
||||||
|
name="Docs",
|
||||||
|
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/docs-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="ops",
|
||||||
|
repo="govoplan-ops",
|
||||||
|
python_package="govoplan-ops",
|
||||||
|
name="Ops",
|
||||||
|
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/ops-webui",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
||||||
|
parser.add_argument("--channel", default="stable")
|
||||||
|
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
||||||
|
parser.add_argument("--expires-days", type=int, default=90)
|
||||||
|
parser.add_argument("--catalog-output", type=Path, required=True)
|
||||||
|
parser.add_argument("--keyring-output", type=Path)
|
||||||
|
parser.add_argument(
|
||||||
|
"--catalog-signing-key",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
metavar="KEY_ID=PRIVATE_KEY",
|
||||||
|
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
||||||
|
parser.add_argument("--repository-base", default=GITEA_BASE)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
version = args.version.removeprefix("v")
|
||||||
|
tag = f"v{version}"
|
||||||
|
generated_at = datetime.now(tz=UTC)
|
||||||
|
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||||
|
expires_at = generated_at + timedelta(days=args.expires_days)
|
||||||
|
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||||
|
|
||||||
|
catalog = _catalog_payload(
|
||||||
|
version=version,
|
||||||
|
tag=tag,
|
||||||
|
channel=args.channel,
|
||||||
|
sequence=sequence,
|
||||||
|
generated_at=generated_at,
|
||||||
|
expires_at=expires_at,
|
||||||
|
repository_base=args.repository_base.rstrip("/"),
|
||||||
|
public_base_url=args.public_base_url.rstrip("/"),
|
||||||
|
)
|
||||||
|
if signing_keys:
|
||||||
|
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
||||||
|
|
||||||
|
output = args.catalog_output.expanduser()
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
if args.keyring_output is not None:
|
||||||
|
keyring_output = args.keyring_output.expanduser()
|
||||||
|
keyring_output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
keyring_output.write_text(
|
||||||
|
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"catalog={output}")
|
||||||
|
if args.keyring_output is not None:
|
||||||
|
print(f"keyring={args.keyring_output.expanduser()}")
|
||||||
|
print(f"channel={args.channel}")
|
||||||
|
print(f"sequence={sequence}")
|
||||||
|
print(f"version={version}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_payload(
|
||||||
|
*,
|
||||||
|
version: str,
|
||||||
|
tag: str,
|
||||||
|
channel: str,
|
||||||
|
sequence: int,
|
||||||
|
generated_at: datetime,
|
||||||
|
expires_at: datetime,
|
||||||
|
repository_base: str,
|
||||||
|
public_base_url: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
manifests = _discovered_catalog_manifests()
|
||||||
|
modules: list[dict[str, Any]] = []
|
||||||
|
for module in CATALOG_MODULES:
|
||||||
|
manifest = manifests.get(module.module_id)
|
||||||
|
module_version = manifest.version if manifest is not None else version
|
||||||
|
module_tag = f"v{module_version.removeprefix('v')}"
|
||||||
|
entry: dict[str, Any] = {
|
||||||
|
"module_id": module.module_id,
|
||||||
|
"name": module.name,
|
||||||
|
"description": module.description,
|
||||||
|
"version": module_version,
|
||||||
|
"action": "install",
|
||||||
|
"python_package": module.python_package,
|
||||||
|
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
|
||||||
|
"license_features": [f"module.{module.module_id}"],
|
||||||
|
"tags": list(module.tags),
|
||||||
|
}
|
||||||
|
if module.webui_package:
|
||||||
|
entry["webui_package"] = module.webui_package
|
||||||
|
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
|
||||||
|
manifest_metadata = _manifest_catalog_metadata(manifest)
|
||||||
|
entry.update(manifest_metadata)
|
||||||
|
if module.provides_interfaces:
|
||||||
|
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
|
||||||
|
if module.requires_interfaces:
|
||||||
|
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
|
||||||
|
modules.append(entry)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"catalog_version": "1",
|
||||||
|
"channel": channel,
|
||||||
|
"sequence": sequence,
|
||||||
|
"generated_at": _json_datetime(generated_at),
|
||||||
|
"expires_at": _json_datetime(expires_at),
|
||||||
|
"release": {
|
||||||
|
"version": version,
|
||||||
|
"tag": tag,
|
||||||
|
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
||||||
|
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
||||||
|
},
|
||||||
|
"core_release": {
|
||||||
|
"name": "GovOPlaN Core",
|
||||||
|
"version": version,
|
||||||
|
"python_package": "govoplan-core",
|
||||||
|
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
||||||
|
"webui_package": "@govoplan/core-webui",
|
||||||
|
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
||||||
|
},
|
||||||
|
"modules": modules,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
|
||||||
|
try:
|
||||||
|
return available_module_manifests(ignore_load_errors=True)
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
|
||||||
|
if manifest is None:
|
||||||
|
return {}
|
||||||
|
payload: dict[str, object] = {}
|
||||||
|
if manifest.dependencies:
|
||||||
|
payload["dependencies"] = list(manifest.dependencies)
|
||||||
|
if manifest.optional_dependencies:
|
||||||
|
payload["optional_dependencies"] = list(manifest.optional_dependencies)
|
||||||
|
if manifest.migration_spec is not None:
|
||||||
|
payload["migration_safety"] = "requires_review"
|
||||||
|
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
|
||||||
|
if manifest.migration_spec.migration_after:
|
||||||
|
payload["migration_after"] = list(manifest.migration_spec.migration_after)
|
||||||
|
if manifest.migration_spec.migration_before:
|
||||||
|
payload["migration_before"] = list(manifest.migration_spec.migration_before)
|
||||||
|
if manifest.migration_spec.migration_tasks:
|
||||||
|
tasks: list[dict[str, object]] = []
|
||||||
|
for task in manifest.migration_spec.migration_tasks:
|
||||||
|
task_payload: dict[str, object] = {
|
||||||
|
"task_id": task.task_id,
|
||||||
|
"phase": task.phase,
|
||||||
|
"summary": task.summary,
|
||||||
|
"task_version": task.task_version,
|
||||||
|
"safety": task.safety,
|
||||||
|
"idempotent": task.idempotent,
|
||||||
|
}
|
||||||
|
if task.timeout_seconds is not None:
|
||||||
|
task_payload["timeout_seconds"] = task.timeout_seconds
|
||||||
|
tasks.append(task_payload)
|
||||||
|
payload["migration_tasks"] = tasks
|
||||||
|
if manifest.provides_interfaces:
|
||||||
|
payload["provides_interfaces"] = [
|
||||||
|
{"name": item.name, "version": item.version}
|
||||||
|
for item in manifest.provides_interfaces
|
||||||
|
]
|
||||||
|
if manifest.requires_interfaces:
|
||||||
|
requirements: list[dict[str, object]] = []
|
||||||
|
for item in manifest.requires_interfaces:
|
||||||
|
requirement: dict[str, object] = {
|
||||||
|
"name": item.name,
|
||||||
|
"optional": item.optional,
|
||||||
|
}
|
||||||
|
if item.version_min is not None:
|
||||||
|
requirement["version_min"] = item.version_min
|
||||||
|
if item.version_max_exclusive is not None:
|
||||||
|
requirement["version_max_exclusive"] = item.version_max_exclusive
|
||||||
|
requirements.append(requirement)
|
||||||
|
payload["requires_interfaces"] = requirements
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
||||||
|
key_id, separator, path_text = value.partition("=")
|
||||||
|
if not separator or not key_id.strip() or not path_text.strip():
|
||||||
|
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||||
|
path = Path(path_text).expanduser()
|
||||||
|
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||||
|
if not isinstance(private_key, Ed25519PrivateKey):
|
||||||
|
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||||
|
return key_id.strip(), private_key
|
||||||
|
|
||||||
|
|
||||||
|
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
|
||||||
|
signature_payload = dict(payload)
|
||||||
|
signature_payload.pop("signature", None)
|
||||||
|
signature_payload.pop("signatures", None)
|
||||||
|
signature = private_key.sign(_canonical_bytes(signature_payload))
|
||||||
|
return {
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": key_id,
|
||||||
|
"value": base64.b64encode(signature).decode("ascii"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"keyring_version": "1",
|
||||||
|
"purpose": "govoplan module package catalog signatures",
|
||||||
|
"generated_at": _json_datetime(generated_at),
|
||||||
|
"keys": [
|
||||||
|
{
|
||||||
|
"key_id": key_id,
|
||||||
|
"status": "active",
|
||||||
|
"public_key": _public_key_base64(private_key),
|
||||||
|
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
||||||
|
}
|
||||||
|
for key_id, private_key in signing_keys
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
||||||
|
public_bytes = private_key.public_key().public_bytes(
|
||||||
|
encoding=serialization.Encoding.Raw,
|
||||||
|
format=serialization.PublicFormat.Raw,
|
||||||
|
)
|
||||||
|
return base64.b64encode(public_bytes).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_bytes(payload: object) -> bytes:
|
||||||
|
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _json_datetime(value: datetime) -> str:
|
||||||
|
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
107
scripts/generate-release-lock.sh
Normal file
107
scripts/generate-release-lock.sh
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/generate-release-lock.sh [options]
|
||||||
|
|
||||||
|
Generates webui/package-lock.release.json from webui/package.release.json in a
|
||||||
|
temporary workspace. The normal development package.json and package-lock.json
|
||||||
|
are left untouched.
|
||||||
|
|
||||||
|
Run this after the module git tags referenced by package.release.json exist and
|
||||||
|
are reachable, and before tagging the core release commit that should contain
|
||||||
|
the regenerated release lockfile.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--npm <path> npm executable to use.
|
||||||
|
-h, --help Show this help.
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
echo "error: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
WEBUI="$ROOT/webui"
|
||||||
|
NPM_BIN="${NPM:-}"
|
||||||
|
NODE_BIN="${NODE:-}"
|
||||||
|
|
||||||
|
if [[ -z "$NPM_BIN" ]]; then
|
||||||
|
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||||
|
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||||
|
else
|
||||||
|
NPM_BIN="npm"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [[ -z "$NODE_BIN" ]]; then
|
||||||
|
if [[ -x "$(dirname "$NPM_BIN")/node" ]]; then
|
||||||
|
NODE_BIN="$(dirname "$NPM_BIN")/node"
|
||||||
|
else
|
||||||
|
NODE_BIN="node"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--npm)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
NPM_BIN="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown argument: $1" >&2
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[[ -f "$WEBUI/package.release.json" ]] || fail "missing $WEBUI/package.release.json"
|
||||||
|
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm executable not found: $NPM_BIN"
|
||||||
|
command -v "$NODE_BIN" >/dev/null 2>&1 || fail "node executable not found: $NODE_BIN"
|
||||||
|
|
||||||
|
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-lock.XXXXXXXX")"
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"
|
||||||
|
if [[ -f "$WEBUI/package-lock.release.json" ]]; then
|
||||||
|
cp "$WEBUI/package-lock.release.json" "$TMP_DIR/package-lock.json"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Generating release lockfile from $WEBUI/package.release.json"
|
||||||
|
echo "Temporary workspace: $TMP_DIR"
|
||||||
|
(
|
||||||
|
cd "$TMP_DIR"
|
||||||
|
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||||
|
mapfile -t GIT_PACKAGES < <(
|
||||||
|
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
|
||||||
|
const fs = require("fs");
|
||||||
|
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||||
|
for (const group of ["dependencies", "devDependencies", "optionalDependencies"]) {
|
||||||
|
for (const [name, spec] of Object.entries(pkg[group] || {})) {
|
||||||
|
if (typeof spec === "string" && spec.startsWith("git+")) {
|
||||||
|
console.log(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NODE
|
||||||
|
)
|
||||||
|
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
|
||||||
|
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
|
||||||
|
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
||||||
|
fi
|
||||||
|
)
|
||||||
|
|
||||||
|
cp "$TMP_DIR/package-lock.json" "$WEBUI/package-lock.release.json"
|
||||||
|
echo "Updated $WEBUI/package-lock.release.json"
|
||||||
580
scripts/gitea-backlog-import.py
Normal file
580
scripts/gitea-backlog-import.py
Normal file
@@ -0,0 +1,580 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Import markdown backlog/planning files into Gitea issues."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import dataclasses
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||||
|
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||||
|
|
||||||
|
REPO_ROOTS = {
|
||||||
|
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
||||||
|
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
||||||
|
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
||||||
|
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
||||||
|
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
||||||
|
"web": pathlib.Path("/mnt/DATA/git/govoplan-web"),
|
||||||
|
}
|
||||||
|
|
||||||
|
MODULE_LABELS = {
|
||||||
|
"core": "module/core",
|
||||||
|
"access": "module/access",
|
||||||
|
"mail": "module/mail",
|
||||||
|
"files": "module/files",
|
||||||
|
"campaign": "module/campaign",
|
||||||
|
"web": "module/web",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class Candidate:
|
||||||
|
repo_key: str
|
||||||
|
title: str
|
||||||
|
body: str
|
||||||
|
labels: tuple[str, ...]
|
||||||
|
fingerprint: str
|
||||||
|
source: str
|
||||||
|
line: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class RepoState:
|
||||||
|
target: Any
|
||||||
|
client: GiteaClient
|
||||||
|
label_ids: dict[str, int]
|
||||||
|
fingerprints: set[str]
|
||||||
|
titles: set[str]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
|
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||||
|
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||||
|
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||||
|
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
load_dotenv(args.env_file)
|
||||||
|
candidates = list(build_candidates(args))
|
||||||
|
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||||
|
|
||||||
|
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
||||||
|
for repo_key, grouped in group_candidates(candidates).items():
|
||||||
|
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
||||||
|
|
||||||
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
if not token:
|
||||||
|
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||||
|
print_preview(candidates)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
||||||
|
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
||||||
|
skipped = len(candidates) - len(missing)
|
||||||
|
|
||||||
|
print(f"Skipped {skipped} existing issue candidate(s).")
|
||||||
|
print(f"Missing {len(missing)} issue candidate(s).")
|
||||||
|
print_preview(missing)
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
created_by_repo: dict[str, list[str]] = {}
|
||||||
|
for candidate in missing:
|
||||||
|
state = states[candidate.repo_key]
|
||||||
|
validate_labels(candidate, state)
|
||||||
|
issue = state.client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||||
|
body={
|
||||||
|
"title": candidate.title,
|
||||||
|
"body": candidate.body,
|
||||||
|
"labels": [state.label_ids[label] for label in candidate.labels],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
number = issue.get("number") or issue.get("index")
|
||||||
|
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
||||||
|
state.fingerprints.add(candidate.fingerprint)
|
||||||
|
state.titles.add(normalize_title(candidate.title))
|
||||||
|
|
||||||
|
print("Created issues:")
|
||||||
|
for repo_key, titles in created_by_repo.items():
|
||||||
|
print(f"{repo_key}:")
|
||||||
|
for title in titles:
|
||||||
|
print(f" {title}")
|
||||||
|
if not created_by_repo:
|
||||||
|
print(" none")
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||||
|
if args.product_backlog.exists():
|
||||||
|
yield from parse_product_backlog(args.product_backlog)
|
||||||
|
if args.access_plan and args.access_plan.exists():
|
||||||
|
yield from parse_access_plan(args.access_plan)
|
||||||
|
if args.web_todo.exists():
|
||||||
|
yield from parse_simple_todo(args.web_todo)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
||||||
|
headings: list[tuple[int, str]] = []
|
||||||
|
status = ""
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
line_number = index + 1
|
||||||
|
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
||||||
|
if heading_match:
|
||||||
|
level = len(heading_match.group(1))
|
||||||
|
title = heading_match.group(2).strip()
|
||||||
|
if level == 2 and title == "Completed Work Archive":
|
||||||
|
break
|
||||||
|
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||||
|
headings.append((level, title))
|
||||||
|
status = ""
|
||||||
|
continue
|
||||||
|
|
||||||
|
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
||||||
|
if status_match:
|
||||||
|
status = status_match.group(1).strip()
|
||||||
|
continue
|
||||||
|
|
||||||
|
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||||
|
if not item_match:
|
||||||
|
continue
|
||||||
|
|
||||||
|
item = collect_continued_item(lines, index, item_match.group(1))
|
||||||
|
section = section_title(headings)
|
||||||
|
repo_key, labels = classify_product_item(item, section)
|
||||||
|
prefix = type_prefix(labels)
|
||||||
|
title = format_title(prefix, item)
|
||||||
|
source = str(path)
|
||||||
|
body = body_for_item(
|
||||||
|
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||||
|
source=source,
|
||||||
|
line=line_number,
|
||||||
|
section=section,
|
||||||
|
status=status,
|
||||||
|
item=item,
|
||||||
|
note="Imported from the consolidated GovOPlaN product backlog.",
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key=repo_key,
|
||||||
|
title=title,
|
||||||
|
body=body,
|
||||||
|
labels=tuple(sorted(set(labels))),
|
||||||
|
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||||
|
source=source,
|
||||||
|
line=line_number,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
lines = text.splitlines()
|
||||||
|
yield from access_stage_candidates(path, lines)
|
||||||
|
yield from access_immediate_candidates(path, lines)
|
||||||
|
yield from access_decision_candidates(path, lines)
|
||||||
|
|
||||||
|
|
||||||
|
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||||
|
index = 0
|
||||||
|
while index < len(lines):
|
||||||
|
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
||||||
|
if not match:
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
stage = match.group(1)
|
||||||
|
name = match.group(2).strip()
|
||||||
|
start_line = index + 1
|
||||||
|
index += 1
|
||||||
|
block: list[str] = []
|
||||||
|
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
||||||
|
block.append(lines[index])
|
||||||
|
index += 1
|
||||||
|
item = f"Access extraction Stage {stage}: {name}"
|
||||||
|
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
||||||
|
body = "\n".join(
|
||||||
|
[
|
||||||
|
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
"Imported from the GovOPlaN access extraction plan.",
|
||||||
|
"",
|
||||||
|
f"- Source: `{path}`",
|
||||||
|
f"- Line: `{start_line}`",
|
||||||
|
f"- Section: `Stage {stage}: {name}`",
|
||||||
|
"",
|
||||||
|
"Plan excerpt:",
|
||||||
|
"",
|
||||||
|
"```markdown",
|
||||||
|
f"### Stage {stage}: {name}",
|
||||||
|
*block,
|
||||||
|
"```",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key="core",
|
||||||
|
title=format_title("[Task]", item),
|
||||||
|
body=body,
|
||||||
|
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=start_line,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||||
|
in_section = False
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
line_number = index + 1
|
||||||
|
if line == "## Immediate Backlog":
|
||||||
|
in_section = True
|
||||||
|
continue
|
||||||
|
if in_section and line.startswith("## "):
|
||||||
|
break
|
||||||
|
if not in_section:
|
||||||
|
continue
|
||||||
|
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||||
|
if not item_match:
|
||||||
|
continue
|
||||||
|
item = collect_continued_item(lines, index, item_match.group(1))
|
||||||
|
title_item = access_title_item(item)
|
||||||
|
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||||
|
body = body_for_item(
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
section="Immediate Backlog",
|
||||||
|
status="open",
|
||||||
|
item=item,
|
||||||
|
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key="core",
|
||||||
|
title=format_title("[Task]", title_item),
|
||||||
|
body=body,
|
||||||
|
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||||
|
in_section = False
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
line_number = index + 1
|
||||||
|
if line == "## Open Decisions":
|
||||||
|
in_section = True
|
||||||
|
continue
|
||||||
|
if in_section and line.startswith("## "):
|
||||||
|
break
|
||||||
|
if not in_section:
|
||||||
|
continue
|
||||||
|
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||||
|
if not item_match:
|
||||||
|
continue
|
||||||
|
item = collect_continued_item(lines, index, item_match.group(1))
|
||||||
|
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||||
|
body = body_for_item(
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
section="Open Decisions",
|
||||||
|
status="decision needed",
|
||||||
|
item=item,
|
||||||
|
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key="core",
|
||||||
|
title=format_title("[Task]", f"Decide: {item}"),
|
||||||
|
body=body,
|
||||||
|
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
line_number = index + 1
|
||||||
|
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||||
|
if not item_match:
|
||||||
|
continue
|
||||||
|
item = collect_continued_item(lines, index, item_match.group(1))
|
||||||
|
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key="web",
|
||||||
|
title=format_title("[Task]", item),
|
||||||
|
body=body_for_item(
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
section="TODO",
|
||||||
|
status="open",
|
||||||
|
item=item,
|
||||||
|
note="Imported from the GovOPlaN website TODO file.",
|
||||||
|
),
|
||||||
|
labels=labels("web", "type/task", "area/marketing"),
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(path),
|
||||||
|
line=line_number,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
||||||
|
text = f"{section} {item}".lower()
|
||||||
|
repo_key = "core"
|
||||||
|
area = "area/devex"
|
||||||
|
issue_type = "type/task"
|
||||||
|
|
||||||
|
if "dsar search/export/delete" in text:
|
||||||
|
return "core", labels("core", "type/feature", "area/governance")
|
||||||
|
if "fully stream zip uploads" in text:
|
||||||
|
return "files", labels("files", "type/task", "area/api")
|
||||||
|
if "installer packaging approach" in text:
|
||||||
|
return "core", labels("core", "type/task", "area/devex")
|
||||||
|
if "profile reselection/revalidation" in text:
|
||||||
|
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
||||||
|
if "module boundaries" in text:
|
||||||
|
return "core", labels("core", issue_type, "area/module-system")
|
||||||
|
if "external storage connectors" in text:
|
||||||
|
repo_key = "files"
|
||||||
|
area = "area/api"
|
||||||
|
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
||||||
|
repo_key = "campaign"
|
||||||
|
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||||
|
elif "controlled sending" in text:
|
||||||
|
repo_key = "campaign"
|
||||||
|
area = "area/api"
|
||||||
|
elif "mail profiles" in text:
|
||||||
|
repo_key = "mail"
|
||||||
|
area = "area/api"
|
||||||
|
|
||||||
|
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
||||||
|
repo_key = "mail"
|
||||||
|
area = "area/api"
|
||||||
|
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
||||||
|
repo_key = "files"
|
||||||
|
area = "area/api"
|
||||||
|
if "external storage connectors" not in text and any(
|
||||||
|
word in text
|
||||||
|
for word in (
|
||||||
|
"campaign",
|
||||||
|
"recipient",
|
||||||
|
"attachment",
|
||||||
|
"zip",
|
||||||
|
"evidence",
|
||||||
|
"wizard",
|
||||||
|
"send",
|
||||||
|
"archive",
|
||||||
|
"message preview",
|
||||||
|
"address book",
|
||||||
|
"carddav",
|
||||||
|
"mailing lists",
|
||||||
|
)
|
||||||
|
):
|
||||||
|
repo_key = "campaign"
|
||||||
|
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||||
|
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
||||||
|
repo_key = "core"
|
||||||
|
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
||||||
|
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
||||||
|
repo_key = "core"
|
||||||
|
area = "area/module-system"
|
||||||
|
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
||||||
|
repo_key = "core"
|
||||||
|
area = "area/release"
|
||||||
|
if repo_key == "core" and not any(
|
||||||
|
phrase in text
|
||||||
|
for phrase in (
|
||||||
|
"external storage connectors",
|
||||||
|
"controlled sending",
|
||||||
|
"report, evidence",
|
||||||
|
"recipient import",
|
||||||
|
"campaign and attachment",
|
||||||
|
)
|
||||||
|
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
||||||
|
repo_key = "core"
|
||||||
|
area = "area/devex"
|
||||||
|
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
||||||
|
area = "area/docs"
|
||||||
|
if "ui lists often start at index 2" in text:
|
||||||
|
issue_type = "type/bug"
|
||||||
|
area = "area/webui"
|
||||||
|
repo_key = "core"
|
||||||
|
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
||||||
|
issue_type = "type/feature"
|
||||||
|
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
||||||
|
issue_type = "type/debt"
|
||||||
|
|
||||||
|
return repo_key, labels(repo_key, issue_type, area)
|
||||||
|
|
||||||
|
|
||||||
|
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
||||||
|
parts = [initial.strip()]
|
||||||
|
next_index = index + 1
|
||||||
|
while next_index < len(lines):
|
||||||
|
line = lines[next_index]
|
||||||
|
if not line.startswith((" ", "\t")):
|
||||||
|
break
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
break
|
||||||
|
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
||||||
|
break
|
||||||
|
parts.append(stripped)
|
||||||
|
next_index += 1
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
||||||
|
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
||||||
|
base.update(extra)
|
||||||
|
if "status/needs-info" in base:
|
||||||
|
base.discard("status/triage")
|
||||||
|
return tuple(sorted(base))
|
||||||
|
|
||||||
|
|
||||||
|
def type_prefix(label_set: Iterable[str]) -> str:
|
||||||
|
labels_set = set(label_set)
|
||||||
|
if "type/bug" in labels_set:
|
||||||
|
return "[Bug]"
|
||||||
|
if "type/feature" in labels_set:
|
||||||
|
return "[Feature]"
|
||||||
|
if "type/debt" in labels_set:
|
||||||
|
return "[Debt]"
|
||||||
|
if "type/docs" in labels_set:
|
||||||
|
return "[Docs]"
|
||||||
|
return "[Task]"
|
||||||
|
|
||||||
|
|
||||||
|
def format_title(prefix: str, item: str) -> str:
|
||||||
|
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
||||||
|
title = f"{prefix} {clean}"
|
||||||
|
if len(title) <= 180:
|
||||||
|
return title
|
||||||
|
return f"{title[:177].rstrip()}..."
|
||||||
|
|
||||||
|
|
||||||
|
def access_title_item(item: str) -> str:
|
||||||
|
lowered = item.lower()
|
||||||
|
if "compatibility wrappers" in lowered:
|
||||||
|
return "Wire core auth compatibility through access capabilities"
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def body_for_item(
|
||||||
|
*,
|
||||||
|
fingerprint: str,
|
||||||
|
source: str,
|
||||||
|
line: int,
|
||||||
|
section: str,
|
||||||
|
status: str,
|
||||||
|
item: str,
|
||||||
|
note: str,
|
||||||
|
) -> str:
|
||||||
|
lines = [
|
||||||
|
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
note,
|
||||||
|
"",
|
||||||
|
f"- Source: `{source}`",
|
||||||
|
f"- Line: `{line}`",
|
||||||
|
f"- Section: `{section}`",
|
||||||
|
]
|
||||||
|
if status:
|
||||||
|
lines.append(f"- Source status: `{status}`")
|
||||||
|
lines.extend(
|
||||||
|
[
|
||||||
|
"",
|
||||||
|
"Imported backlog item:",
|
||||||
|
"",
|
||||||
|
"```markdown",
|
||||||
|
f"- [ ] {item}",
|
||||||
|
"```",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||||
|
return " > ".join(title for _, title in headings)
|
||||||
|
|
||||||
|
|
||||||
|
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
||||||
|
stable = "\n".join([source, str(line), repo_key, item])
|
||||||
|
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
||||||
|
grouped: dict[str, list[Candidate]] = {}
|
||||||
|
for candidate in candidates:
|
||||||
|
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||||
|
states: dict[str, RepoState] = {}
|
||||||
|
for repo_key in repo_keys:
|
||||||
|
root = repo_root(REPO_ROOTS[repo_key])
|
||||||
|
target = infer_target(root)
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||||
|
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||||
|
fingerprints: set[str] = set()
|
||||||
|
titles: set[str] = set()
|
||||||
|
for issue in issues:
|
||||||
|
body = str(issue.get("body") or "")
|
||||||
|
if issue.get("state") == "closed" and "Moved to `" in body:
|
||||||
|
continue
|
||||||
|
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||||
|
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
||||||
|
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
||||||
|
return states
|
||||||
|
|
||||||
|
|
||||||
|
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
||||||
|
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_title(title: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
||||||
|
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
||||||
|
if missing:
|
||||||
|
joined = ", ".join(missing)
|
||||||
|
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
||||||
|
|
||||||
|
|
||||||
|
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
||||||
|
for repo_key, grouped in group_candidates(candidates).items():
|
||||||
|
print(f"{repo_key}:")
|
||||||
|
for candidate in grouped[:limit]:
|
||||||
|
print(f" {candidate.title}")
|
||||||
|
if len(grouped) > limit:
|
||||||
|
print(f" ... {len(grouped) - limit} more")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
97
scripts/gitea-codex-note.py
Normal file
97
scripts/gitea-codex-note.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Post a standardized Codex state update comment to a Gitea issue."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||||
|
|
||||||
|
|
||||||
|
STATUS_LABELS = {
|
||||||
|
"started": "status/in-progress",
|
||||||
|
"progress": "status/in-progress",
|
||||||
|
"blocked": "status/blocked",
|
||||||
|
"needs-info": "status/needs-info",
|
||||||
|
"ready": "status/ready",
|
||||||
|
"done": "",
|
||||||
|
"note": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||||
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
|
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
||||||
|
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
||||||
|
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
||||||
|
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
||||||
|
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
||||||
|
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
||||||
|
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
||||||
|
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = repo_root(args.root)
|
||||||
|
load_dotenv(args.env_file or root / ".env")
|
||||||
|
target = infer_target(root, args.remote)
|
||||||
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
body = build_body(args)
|
||||||
|
|
||||||
|
print(f"Target: {target.display}")
|
||||||
|
print(f"Issue: #{args.issue}")
|
||||||
|
print(body)
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
||||||
|
body={"body": body},
|
||||||
|
)
|
||||||
|
if args.close:
|
||||||
|
client.request_json(
|
||||||
|
"PATCH",
|
||||||
|
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
||||||
|
body={"state": "closed"},
|
||||||
|
)
|
||||||
|
print("Posted Codex note.")
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def build_body(args: argparse.Namespace) -> str:
|
||||||
|
lines = [f"## Codex State: {args.status}", ""]
|
||||||
|
append_section(lines, "Summary", args.summary)
|
||||||
|
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
||||||
|
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
||||||
|
append_section(lines, "Next / Blocked", args.next)
|
||||||
|
if STATUS_LABELS[args.status]:
|
||||||
|
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
||||||
|
if args.body_file:
|
||||||
|
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
||||||
|
return "\n".join(lines).rstrip() + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
||||||
|
if not items:
|
||||||
|
return
|
||||||
|
lines.extend([f"### {title}", ""])
|
||||||
|
lines.extend(f"- {item}" for item in items)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
597
scripts/gitea-import-all-backlogs.py
Normal file
597
scripts/gitea-import-all-backlogs.py
Normal file
@@ -0,0 +1,597 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import dataclasses
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||||
|
|
||||||
|
|
||||||
|
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||||
|
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||||
|
EXCLUDED_FILE_PATTERNS = (
|
||||||
|
"/.git/",
|
||||||
|
"/.venv/",
|
||||||
|
"/venv/",
|
||||||
|
"/site-packages/",
|
||||||
|
"/node_modules/",
|
||||||
|
"/__pycache__/",
|
||||||
|
"/dist/",
|
||||||
|
"/build/",
|
||||||
|
"/.cache/",
|
||||||
|
"/.module-test-build/",
|
||||||
|
)
|
||||||
|
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
||||||
|
ACTION_RE = re.compile(
|
||||||
|
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
||||||
|
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
||||||
|
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
SKIP_SECTION_RE = re.compile(
|
||||||
|
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
||||||
|
r"available now|done|archive)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
ACTIVE_SECTION_RE = re.compile(
|
||||||
|
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
||||||
|
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
GENERIC_LABELS = [
|
||||||
|
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
||||||
|
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
||||||
|
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
||||||
|
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
||||||
|
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
||||||
|
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
||||||
|
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
||||||
|
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
||||||
|
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
||||||
|
("priority/p2", "fbca04", "Normal planned priority."),
|
||||||
|
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class RepoInfo:
|
||||||
|
root: pathlib.Path
|
||||||
|
remote: str
|
||||||
|
owner: str
|
||||||
|
repo: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def key(self) -> str:
|
||||||
|
return self.root.name
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class SourceFile:
|
||||||
|
repo_key: str
|
||||||
|
path: pathlib.Path
|
||||||
|
source_kind: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class Candidate:
|
||||||
|
repo_key: str
|
||||||
|
title: str
|
||||||
|
body: str
|
||||||
|
labels: tuple[str, ...]
|
||||||
|
priority: str
|
||||||
|
milestone: str
|
||||||
|
fingerprint: str
|
||||||
|
source: str
|
||||||
|
line: int
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||||
|
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||||
|
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||||
|
parser.add_argument("--apply", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
load_dotenv(args.env_file)
|
||||||
|
repos = discover_hosted_repos(args.git_root)
|
||||||
|
sources = discover_sources(repos, args.products_root)
|
||||||
|
candidates = list(build_candidates(sources))
|
||||||
|
candidates = dedupe_candidates(candidates)
|
||||||
|
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||||
|
|
||||||
|
print(f"Hosted repositories: {len(repos)}")
|
||||||
|
print(f"Backlog-like source files: {len(sources)}")
|
||||||
|
print(f"Issue candidates: {len(candidates)}")
|
||||||
|
for repo_key, count in grouped_counts(candidates).items():
|
||||||
|
print(f" {repo_key}: {count}")
|
||||||
|
|
||||||
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
if not token:
|
||||||
|
print_preview(candidates)
|
||||||
|
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
missing_by_repo: dict[str, list[Candidate]] = {}
|
||||||
|
skipped = 0
|
||||||
|
states: dict[str, RepoState] = {}
|
||||||
|
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
||||||
|
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
||||||
|
for candidate in candidates:
|
||||||
|
state = states[candidate.repo_key]
|
||||||
|
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||||
|
|
||||||
|
missing_total = sum(len(items) for items in missing_by_repo.values())
|
||||||
|
print(f"Skipped existing: {skipped}")
|
||||||
|
print(f"Missing candidates: {missing_total}")
|
||||||
|
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
for repo_key, repo_candidates in missing_by_repo.items():
|
||||||
|
state = states[repo_key]
|
||||||
|
for candidate in repo_candidates:
|
||||||
|
label_ids = [state.label_ids[label] for label in candidate.labels]
|
||||||
|
if candidate.priority not in candidate.labels:
|
||||||
|
label_ids.append(state.label_ids[candidate.priority])
|
||||||
|
milestone_id = ensure_milestone(state, candidate.milestone)
|
||||||
|
created = state.client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||||
|
body={
|
||||||
|
"title": candidate.title,
|
||||||
|
"body": candidate.body,
|
||||||
|
"labels": label_ids,
|
||||||
|
"milestone": milestone_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
number = created.get("number") or created.get("index")
|
||||||
|
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
||||||
|
state.fingerprints.add(candidate.fingerprint)
|
||||||
|
state.titles.add(normalize_title(candidate.title))
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class RepoState:
|
||||||
|
target: Any
|
||||||
|
client: GiteaClient
|
||||||
|
label_ids: dict[str, int]
|
||||||
|
milestone_ids: dict[str, int]
|
||||||
|
fingerprints: set[str]
|
||||||
|
titles: set[str]
|
||||||
|
|
||||||
|
|
||||||
|
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||||
|
repos: dict[str, RepoInfo] = {}
|
||||||
|
for gitdir in sorted(git_root.glob("*/.git")):
|
||||||
|
root = gitdir.parent
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
remote = result.stdout.strip()
|
||||||
|
if "git.add-ideas.de" not in remote:
|
||||||
|
continue
|
||||||
|
target = infer_target(root)
|
||||||
|
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
||||||
|
return repos
|
||||||
|
|
||||||
|
|
||||||
|
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
||||||
|
sources: list[SourceFile] = []
|
||||||
|
for repo in repos.values():
|
||||||
|
for path in repo.root.rglob("*"):
|
||||||
|
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
||||||
|
continue
|
||||||
|
path_text = path.as_posix()
|
||||||
|
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
||||||
|
continue
|
||||||
|
if is_excluded_repo_file(path):
|
||||||
|
continue
|
||||||
|
if is_text_backlog(path):
|
||||||
|
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
||||||
|
|
||||||
|
product_map = {
|
||||||
|
"co2api": "emission-api-lib",
|
||||||
|
"govoplan": "govoplan-core",
|
||||||
|
"meubility": "meubility-workbench",
|
||||||
|
"mousehold": "mousehold",
|
||||||
|
"prvnncr": "prvnncr-server",
|
||||||
|
}
|
||||||
|
if products_root.exists():
|
||||||
|
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||||
|
repo_key = product_map.get(product_dir.name)
|
||||||
|
if not repo_key or repo_key not in repos:
|
||||||
|
continue
|
||||||
|
for path in product_dir.rglob("*"):
|
||||||
|
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
||||||
|
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
||||||
|
|
||||||
|
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
||||||
|
for source in sources:
|
||||||
|
unique[(source.repo_key, source.path.resolve())] = source
|
||||||
|
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
||||||
|
|
||||||
|
|
||||||
|
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||||
|
text = path.as_posix()
|
||||||
|
name = path.name.lower()
|
||||||
|
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
|
||||||
|
return True
|
||||||
|
if "testing_plan" in name or "test_plan" in name:
|
||||||
|
return True
|
||||||
|
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||||
|
return True
|
||||||
|
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||||
|
return True
|
||||||
|
if text.endswith("/govoplan-web/TODO.md"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_text_backlog(path: pathlib.Path) -> bool:
|
||||||
|
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
chunk = path.read_bytes()[:4096]
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if b"\x00" in chunk:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
||||||
|
for source in sources:
|
||||||
|
suffix = source.path.suffix.lower()
|
||||||
|
if suffix == ".csv":
|
||||||
|
yield from parse_csv_source(source)
|
||||||
|
else:
|
||||||
|
yield from parse_text_source(source)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
||||||
|
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||||
|
reader = csv.DictReader(handle)
|
||||||
|
for index, row in enumerate(reader, start=2):
|
||||||
|
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
||||||
|
if not story:
|
||||||
|
continue
|
||||||
|
identifier = (row.get("id") or "").strip()
|
||||||
|
priority = normalize_priority(row.get("priority") or "")
|
||||||
|
milestone = (row.get("epic") or "Backlog").strip()
|
||||||
|
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
||||||
|
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
||||||
|
body = "\n".join(
|
||||||
|
[
|
||||||
|
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
"Imported from a CSV backlog file.",
|
||||||
|
"",
|
||||||
|
f"- Source: `{source.path}`",
|
||||||
|
f"- Line: `{index}`",
|
||||||
|
f"- Source kind: `{source.source_kind}`",
|
||||||
|
f"- Milestone: `{milestone}`",
|
||||||
|
"",
|
||||||
|
"CSV row:",
|
||||||
|
"",
|
||||||
|
"```text",
|
||||||
|
", ".join(f"{key}={value}" for key, value in row.items()),
|
||||||
|
"```",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key=source.repo_key,
|
||||||
|
title=title,
|
||||||
|
body=body,
|
||||||
|
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
||||||
|
priority=priority,
|
||||||
|
milestone=milestone,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(source.path),
|
||||||
|
line=index,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
||||||
|
try:
|
||||||
|
lines = source.path.read_text(encoding="utf-8").splitlines()
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
lines = source.path.read_text(encoding="latin-1").splitlines()
|
||||||
|
|
||||||
|
headings: list[tuple[int, str]] = []
|
||||||
|
in_tasks = False
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
line_number = index + 1
|
||||||
|
heading = parse_heading(line)
|
||||||
|
if heading:
|
||||||
|
level, title = heading
|
||||||
|
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||||
|
headings.append((level, title))
|
||||||
|
in_tasks = False
|
||||||
|
continue
|
||||||
|
|
||||||
|
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
||||||
|
in_tasks = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
item = parse_open_item(line)
|
||||||
|
if item is None and is_action_context(source.path, headings, in_tasks):
|
||||||
|
item = parse_plain_action_item(line)
|
||||||
|
if item is None:
|
||||||
|
continue
|
||||||
|
if not item or should_skip_item(item, headings):
|
||||||
|
continue
|
||||||
|
|
||||||
|
section = section_title(headings)
|
||||||
|
priority = priority_from_context(section, item)
|
||||||
|
milestone = milestone_from_context(source.path, section)
|
||||||
|
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
||||||
|
title = format_title(prefix, item)
|
||||||
|
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
||||||
|
body = "\n".join(
|
||||||
|
[
|
||||||
|
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
"Imported from a backlog-like text file.",
|
||||||
|
"",
|
||||||
|
f"- Source: `{source.path}`",
|
||||||
|
f"- Line: `{line_number}`",
|
||||||
|
f"- Source kind: `{source.source_kind}`",
|
||||||
|
f"- Section: `{section or 'none'}`",
|
||||||
|
"",
|
||||||
|
"Imported item:",
|
||||||
|
"",
|
||||||
|
"```text",
|
||||||
|
item,
|
||||||
|
"```",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
yield Candidate(
|
||||||
|
repo_key=source.repo_key,
|
||||||
|
title=title,
|
||||||
|
body=body,
|
||||||
|
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
||||||
|
priority=priority,
|
||||||
|
milestone=milestone,
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
source=str(source.path),
|
||||||
|
line=line_number,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_heading(line: str) -> tuple[int, str] | None:
|
||||||
|
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
||||||
|
if markdown:
|
||||||
|
return len(markdown.group(1)), clean_text(markdown.group(2))
|
||||||
|
underlined = re.match(r"^(.+?)\s*$", line)
|
||||||
|
if underlined and line.strip() and len(line.strip()) < 120:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_open_item(line: str) -> str | None:
|
||||||
|
patterns = [
|
||||||
|
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
||||||
|
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
||||||
|
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
||||||
|
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.match(pattern, line)
|
||||||
|
if match and leading_width(match.group("indent")) == 0:
|
||||||
|
return clean_text(match.group("text"))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_plain_action_item(line: str) -> str | None:
|
||||||
|
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
||||||
|
if match and leading_width(match.group("indent")) == 0:
|
||||||
|
text = clean_text(match.group("text"))
|
||||||
|
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
||||||
|
return None
|
||||||
|
if ACTION_RE.match(text) or is_short_noun_task(text):
|
||||||
|
return text
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
||||||
|
if in_tasks:
|
||||||
|
return True
|
||||||
|
context = section_title(headings)
|
||||||
|
if not context:
|
||||||
|
return False
|
||||||
|
if SKIP_SECTION_RE.search(context):
|
||||||
|
return False
|
||||||
|
if ACTIVE_SECTION_RE.search(context):
|
||||||
|
return True
|
||||||
|
return path.name.lower() in {"todo.txt", "todo.md"}
|
||||||
|
|
||||||
|
|
||||||
|
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
||||||
|
context = section_title(headings)
|
||||||
|
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
||||||
|
return True
|
||||||
|
if item.endswith((",", ";")):
|
||||||
|
return True
|
||||||
|
if SKIP_SECTION_RE.search(context):
|
||||||
|
return True
|
||||||
|
if len(item) < 4:
|
||||||
|
return True
|
||||||
|
if item.endswith(":") and len(item.split()) <= 6:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_short_noun_task(text: str) -> bool:
|
||||||
|
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_text(text: str) -> str:
|
||||||
|
text = text.strip()
|
||||||
|
text = re.sub(r"^☐\s*", "", text)
|
||||||
|
text = re.sub(r"^✅\s*", "", text)
|
||||||
|
text = re.sub(r"\s+", " ", text)
|
||||||
|
return text.strip(" -")
|
||||||
|
|
||||||
|
|
||||||
|
def leading_width(indent: str) -> int:
|
||||||
|
return len(indent.replace("\t", " "))
|
||||||
|
|
||||||
|
|
||||||
|
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||||
|
return " > ".join(title for _, title in headings)
|
||||||
|
|
||||||
|
|
||||||
|
def priority_from_context(section: str, item: str) -> str:
|
||||||
|
text = f"{section} {item}".lower()
|
||||||
|
if "p0" in text or "critical path" in text:
|
||||||
|
return "priority/p0"
|
||||||
|
if "p1" in text or "immediate" in text or "next sprint" in text:
|
||||||
|
return "priority/p1"
|
||||||
|
if "p2" in text:
|
||||||
|
return "priority/p2"
|
||||||
|
if "p3" in text or "future" in text or "later" in text:
|
||||||
|
return "priority/p3"
|
||||||
|
return "priority/p2"
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_priority(value: str) -> str:
|
||||||
|
value = value.strip().lower()
|
||||||
|
if value == "p0":
|
||||||
|
return "priority/p0"
|
||||||
|
if value == "p1":
|
||||||
|
return "priority/p1"
|
||||||
|
if value == "p2":
|
||||||
|
return "priority/p2"
|
||||||
|
if value == "p3":
|
||||||
|
return "priority/p3"
|
||||||
|
return "priority/p2"
|
||||||
|
|
||||||
|
|
||||||
|
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
||||||
|
if section:
|
||||||
|
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
||||||
|
if parts:
|
||||||
|
return parts[-1][:120]
|
||||||
|
return path.stem.replace("_", " ").replace("-", " ").title()
|
||||||
|
|
||||||
|
|
||||||
|
def format_title(prefix: str, item: str) -> str:
|
||||||
|
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
||||||
|
if len(title) <= 180:
|
||||||
|
return title
|
||||||
|
return f"{title[:177].rstrip()}..."
|
||||||
|
|
||||||
|
|
||||||
|
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
||||||
|
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
||||||
|
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
||||||
|
seen: set[tuple[str, str]] = set()
|
||||||
|
unique: list[Candidate] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
key = (candidate.repo_key, normalize_title(candidate.title))
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
unique.append(candidate)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
|
||||||
|
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
for candidate in candidates:
|
||||||
|
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
||||||
|
return dict(sorted(counts.items()))
|
||||||
|
|
||||||
|
|
||||||
|
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||||
|
target = infer_target(repo.root)
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||||
|
if apply:
|
||||||
|
for name, color, description in GENERIC_LABELS:
|
||||||
|
if name in label_ids:
|
||||||
|
continue
|
||||||
|
created = client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(target.owner, target.repo, "/labels"),
|
||||||
|
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
||||||
|
)
|
||||||
|
label_ids[str(created["name"])] = int(created["id"])
|
||||||
|
print(f"created label {target.owner}/{target.repo}:{name}")
|
||||||
|
|
||||||
|
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
||||||
|
if missing and apply:
|
||||||
|
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
||||||
|
|
||||||
|
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
||||||
|
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
||||||
|
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
||||||
|
fingerprints: set[str] = set()
|
||||||
|
titles: set[str] = set()
|
||||||
|
for issue in issues:
|
||||||
|
body = str(issue.get("body") or "")
|
||||||
|
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||||
|
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
||||||
|
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_milestone(state: RepoState, title: str) -> int:
|
||||||
|
if title in state.milestone_ids:
|
||||||
|
return state.milestone_ids[title]
|
||||||
|
created = state.client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
||||||
|
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
||||||
|
)
|
||||||
|
state.milestone_ids[title] = int(created["id"])
|
||||||
|
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
||||||
|
return state.milestone_ids[title]
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_title(title: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
||||||
|
by_repo: dict[str, list[Candidate]] = {}
|
||||||
|
for candidate in candidates:
|
||||||
|
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||||
|
for repo_key, items in sorted(by_repo.items()):
|
||||||
|
print(f"{repo_key}:")
|
||||||
|
for item in items[:limit_per_repo]:
|
||||||
|
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
||||||
|
if len(items) > limit_per_repo:
|
||||||
|
print(f" ... {len(items) - limit_per_repo} more")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
113
scripts/gitea-install-workflow.py
Normal file
113
scripts/gitea-install-workflow.py
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Install Gitea issue workflow files into one or more repositories."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from gitea_common import GiteaError, repo_root
|
||||||
|
|
||||||
|
|
||||||
|
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||||
|
WORKFLOW_FILES = (
|
||||||
|
".gitea/PULL_REQUEST_TEMPLATE.md",
|
||||||
|
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
||||||
|
".gitea/ISSUE_TEMPLATE/config.yaml",
|
||||||
|
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
||||||
|
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
||||||
|
".gitea/ISSUE_TEMPLATE/task.md",
|
||||||
|
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
||||||
|
)
|
||||||
|
MODULE_LABEL_BY_REPO = {
|
||||||
|
"govoplan-core": "module/core",
|
||||||
|
"govoplan-access": "module/access",
|
||||||
|
"govoplan-admin": "module/admin",
|
||||||
|
"govoplan-tenancy": "module/tenancy",
|
||||||
|
"govoplan-policy": "module/policy",
|
||||||
|
"govoplan-audit": "module/audit",
|
||||||
|
"govoplan-mail": "module/mail",
|
||||||
|
"govoplan-files": "module/files",
|
||||||
|
"govoplan-campaign": "module/campaign",
|
||||||
|
"govoplan-web": "module/web",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
||||||
|
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
||||||
|
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
||||||
|
unique_roots = list(dict.fromkeys(target_roots))
|
||||||
|
if args.module_label and len(unique_roots) != 1:
|
||||||
|
raise GiteaError("--module-label can only be used with a single target")
|
||||||
|
|
||||||
|
rel_paths = list(WORKFLOW_FILES)
|
||||||
|
if args.include_labels_file:
|
||||||
|
rel_paths.append("docs/gitea-labels.json")
|
||||||
|
|
||||||
|
for target_root in unique_roots:
|
||||||
|
module_label = args.module_label or infer_module_label(target_root)
|
||||||
|
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
print("Dry run only. Re-run with --apply to write files.")
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def infer_module_label(target_root: pathlib.Path) -> str:
|
||||||
|
repo_name = target_root.name
|
||||||
|
try:
|
||||||
|
repo_name = target_root.resolve().name
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
||||||
|
if label:
|
||||||
|
return label
|
||||||
|
if repo_name.startswith("govoplan-"):
|
||||||
|
suffix = repo_name.removeprefix("govoplan-")
|
||||||
|
if suffix:
|
||||||
|
return f"module/{suffix}"
|
||||||
|
raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/<name>.")
|
||||||
|
|
||||||
|
|
||||||
|
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
||||||
|
print(f"Target: {target_root} ({module_label})")
|
||||||
|
for rel_path in rel_paths:
|
||||||
|
source = SOURCE_ROOT / rel_path
|
||||||
|
target = target_root / rel_path
|
||||||
|
if not source.exists():
|
||||||
|
raise GiteaError(f"missing workflow source file: {source}")
|
||||||
|
|
||||||
|
content = source.read_text(encoding="utf-8")
|
||||||
|
content = transform_content(rel_path, content, module_label)
|
||||||
|
|
||||||
|
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
||||||
|
if existing == content:
|
||||||
|
print(f"unchanged {rel_path}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
action = "create" if existing is None else "update"
|
||||||
|
print(f"{action if apply else 'would ' + action} {rel_path}")
|
||||||
|
if apply:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text(content, encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
||||||
|
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
||||||
|
return content.replace(" - module/core", f" - {module_label}")
|
||||||
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
250
scripts/gitea-migrate-org-labels.py
Normal file
250
scripts/gitea-migrate-org-labels.py
Normal file
@@ -0,0 +1,250 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Move issue labels from repository-local labels to organization labels."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from gitea_common import (
|
||||||
|
GiteaClient,
|
||||||
|
GiteaError,
|
||||||
|
infer_target,
|
||||||
|
load_dotenv,
|
||||||
|
org_path,
|
||||||
|
repo_path,
|
||||||
|
repo_root,
|
||||||
|
require_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||||
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||||
|
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||||
|
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||||
|
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||||
|
parser.add_argument(
|
||||||
|
"--issue-mode",
|
||||||
|
choices=("replace", "delete-add"),
|
||||||
|
default="replace",
|
||||||
|
help="replace labels in one request, or delete local ids before adding org ids",
|
||||||
|
)
|
||||||
|
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = repo_root(args.root)
|
||||||
|
load_dotenv(args.env_file or root / ".env")
|
||||||
|
target = infer_target(root, args.remote)
|
||||||
|
owner = args.org or target.owner
|
||||||
|
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||||
|
if client.token is None:
|
||||||
|
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||||
|
|
||||||
|
repo_filter = re.compile(args.repo_regex)
|
||||||
|
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||||
|
if not org_labels:
|
||||||
|
raise GiteaError(f"{owner} has no organization labels")
|
||||||
|
|
||||||
|
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||||
|
totals = Totals()
|
||||||
|
print(f"Organization: {owner}")
|
||||||
|
print(f"Repositories: {len(repos)}")
|
||||||
|
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||||
|
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||||
|
|
||||||
|
for index, repo in enumerate(repos, start=1):
|
||||||
|
repo_result = migrate_repository(
|
||||||
|
client,
|
||||||
|
owner=owner,
|
||||||
|
repo=repo,
|
||||||
|
org_labels=org_labels,
|
||||||
|
issue_mode=args.issue_mode,
|
||||||
|
apply=args.apply,
|
||||||
|
delete_repo_labels=args.delete_repo_labels,
|
||||||
|
)
|
||||||
|
totals.add(repo_result)
|
||||||
|
if repo_result.has_work:
|
||||||
|
print(
|
||||||
|
f"[{index}/{len(repos)}] {repo}: "
|
||||||
|
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||||
|
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||||
|
f"errors={len(repo_result.errors)}"
|
||||||
|
)
|
||||||
|
for error in repo_result.errors:
|
||||||
|
print(f" error: {error}")
|
||||||
|
else:
|
||||||
|
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||||
|
|
||||||
|
print("Summary:")
|
||||||
|
print(f" repositories processed: {totals.repositories}")
|
||||||
|
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||||
|
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||||
|
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||||
|
print(f" errors: {totals.errors}")
|
||||||
|
if totals.errors:
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
class RepoResult:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.issue_label_migrations = 0
|
||||||
|
self.repo_label_deletions = 0
|
||||||
|
self.errors: list[str] = []
|
||||||
|
self.matched_local_labels = 0
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_work(self) -> bool:
|
||||||
|
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||||
|
|
||||||
|
|
||||||
|
class Totals:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.repositories = 0
|
||||||
|
self.repositories_with_work = 0
|
||||||
|
self.issue_label_migrations = 0
|
||||||
|
self.repo_label_deletions = 0
|
||||||
|
self.errors = 0
|
||||||
|
|
||||||
|
def add(self, result: RepoResult) -> None:
|
||||||
|
self.repositories += 1
|
||||||
|
if result.has_work:
|
||||||
|
self.repositories_with_work += 1
|
||||||
|
self.issue_label_migrations += result.issue_label_migrations
|
||||||
|
self.repo_label_deletions += result.repo_label_deletions
|
||||||
|
self.errors += len(result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_repository(
|
||||||
|
client: GiteaClient,
|
||||||
|
*,
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
org_labels: dict[str, dict[str, Any]],
|
||||||
|
issue_mode: str,
|
||||||
|
apply: bool,
|
||||||
|
delete_repo_labels: bool,
|
||||||
|
) -> RepoResult:
|
||||||
|
result = RepoResult()
|
||||||
|
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||||
|
local_labels = {
|
||||||
|
str(label.get("name") or ""): label
|
||||||
|
for label in repo_labels
|
||||||
|
if str(label.get("name") or "") in org_labels
|
||||||
|
}
|
||||||
|
result.matched_local_labels = len(local_labels)
|
||||||
|
if not local_labels:
|
||||||
|
return result
|
||||||
|
|
||||||
|
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||||
|
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||||
|
|
||||||
|
for issue_type in ("issues", "pulls"):
|
||||||
|
issues = client.paginate(
|
||||||
|
repo_path(owner, repo, "/issues"),
|
||||||
|
query={"state": "all", "type": issue_type},
|
||||||
|
limit=100,
|
||||||
|
)
|
||||||
|
for issue in issues:
|
||||||
|
issue_number = int(issue.get("number") or issue.get("index"))
|
||||||
|
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||||
|
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||||
|
final_label_ids = list(issue_label_ids)
|
||||||
|
changed = False
|
||||||
|
migrated_count = 0
|
||||||
|
local_ids_to_remove: list[int] = []
|
||||||
|
org_ids_to_add: list[int] = []
|
||||||
|
for label in issue.get("labels") or []:
|
||||||
|
local_id = _label_id(label)
|
||||||
|
if local_id is None or local_id not in local_by_id:
|
||||||
|
continue
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
org_id = org_by_name.get(name)
|
||||||
|
if org_id is None:
|
||||||
|
continue
|
||||||
|
local_ids_to_remove.append(local_id)
|
||||||
|
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||||
|
org_ids_to_add.append(org_id)
|
||||||
|
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||||
|
if org_id not in final_label_ids:
|
||||||
|
final_label_ids.append(org_id)
|
||||||
|
changed = True
|
||||||
|
migrated_count += 1
|
||||||
|
if changed:
|
||||||
|
if apply:
|
||||||
|
if issue_mode == "delete-add":
|
||||||
|
for local_id in local_ids_to_remove:
|
||||||
|
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||||
|
if org_ids_to_add:
|
||||||
|
client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||||
|
body={"labels": org_ids_to_add},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
client.request_json(
|
||||||
|
"PUT",
|
||||||
|
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||||
|
body={"labels": final_label_ids},
|
||||||
|
)
|
||||||
|
result.issue_label_migrations += migrated_count
|
||||||
|
|
||||||
|
if delete_repo_labels:
|
||||||
|
for name, label in sorted(local_labels.items()):
|
||||||
|
label_id = _label_id(label)
|
||||||
|
if label_id is None:
|
||||||
|
result.errors.append(f"{name} has no label id")
|
||||||
|
continue
|
||||||
|
if apply:
|
||||||
|
try:
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||||
|
except GiteaError as exc:
|
||||||
|
result.errors.append(f"delete repository label {name}: {exc}")
|
||||||
|
continue
|
||||||
|
result.repo_label_deletions += 1
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||||
|
if explicit:
|
||||||
|
return sorted(set(explicit))
|
||||||
|
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||||
|
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||||
|
return [name for name in names if repo_filter.search(name)]
|
||||||
|
|
||||||
|
|
||||||
|
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||||
|
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||||
|
|
||||||
|
|
||||||
|
def _label_id(label: dict[str, Any]) -> int | None:
|
||||||
|
value = label.get("id") or label.get("index")
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return int(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||||
|
try:
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||||
|
except GiteaError as exc:
|
||||||
|
if "HTTP 404" in str(exc):
|
||||||
|
return
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
180
scripts/gitea-sync-labels.py
Normal file
180
scripts/gitea-sync-labels.py
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Synchronize issue labels to a Gitea repository."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from gitea_common import (
|
||||||
|
GiteaClient,
|
||||||
|
GiteaError,
|
||||||
|
infer_target,
|
||||||
|
load_dotenv,
|
||||||
|
load_json,
|
||||||
|
org_path,
|
||||||
|
repo_path,
|
||||||
|
repo_root,
|
||||||
|
require_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||||
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
|
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
|
parser.add_argument(
|
||||||
|
"--scope",
|
||||||
|
choices=("repository", "organization"),
|
||||||
|
default="repository",
|
||||||
|
help="sync repository labels or organization labels",
|
||||||
|
)
|
||||||
|
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = repo_root(args.root)
|
||||||
|
load_dotenv(args.env_file or root / ".env")
|
||||||
|
target = infer_target(root, args.remote)
|
||||||
|
labels = _load_labels(args.labels_file)
|
||||||
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
org_name = args.org or target.owner
|
||||||
|
|
||||||
|
if args.scope == "organization":
|
||||||
|
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||||
|
else:
|
||||||
|
print(f"Target: {target.display}")
|
||||||
|
print(f"Labels file: {args.labels_file}")
|
||||||
|
|
||||||
|
if not args.apply and not token:
|
||||||
|
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
||||||
|
for label in labels:
|
||||||
|
print(f"would ensure {label['name']} ({label['color']})")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
if args.scope == "organization":
|
||||||
|
existing = _org_labels_by_name(client, org_name)
|
||||||
|
else:
|
||||||
|
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||||
|
creates: list[dict[str, Any]] = []
|
||||||
|
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||||
|
|
||||||
|
for label in labels:
|
||||||
|
current = existing.get(label["name"])
|
||||||
|
if current is None:
|
||||||
|
creates.append(label)
|
||||||
|
continue
|
||||||
|
|
||||||
|
patch = _diff_label(current, label)
|
||||||
|
if patch:
|
||||||
|
updates.append((current, patch))
|
||||||
|
|
||||||
|
if not creates and not updates:
|
||||||
|
print("No label changes needed.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
for label in creates:
|
||||||
|
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||||
|
if args.apply:
|
||||||
|
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||||
|
|
||||||
|
for current, patch in updates:
|
||||||
|
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||||
|
if args.apply:
|
||||||
|
label_id = current.get("id") or current.get("index")
|
||||||
|
if label_id is None:
|
||||||
|
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||||
|
client.request_json(
|
||||||
|
"PATCH",
|
||||||
|
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||||
|
body=patch,
|
||||||
|
)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||||
|
payload = load_json(path)
|
||||||
|
if not isinstance(payload, list):
|
||||||
|
raise GiteaError(f"{path} must contain a JSON list")
|
||||||
|
|
||||||
|
labels: list[dict[str, Any]] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for index, item in enumerate(payload, start=1):
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
raise GiteaError(f"label #{index} must be an object")
|
||||||
|
label = {
|
||||||
|
"name": str(item.get("name", "")).strip(),
|
||||||
|
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
||||||
|
"description": str(item.get("description", "")).strip(),
|
||||||
|
"exclusive": bool(item.get("exclusive", False)),
|
||||||
|
}
|
||||||
|
if not label["name"]:
|
||||||
|
raise GiteaError(f"label #{index} is missing name")
|
||||||
|
if label["name"] in seen:
|
||||||
|
raise GiteaError(f"duplicate label name: {label['name']}")
|
||||||
|
if not _is_hex_color(label["color"]):
|
||||||
|
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
||||||
|
seen.add(label["name"])
|
||||||
|
labels.append(label)
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||||
|
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||||
|
return {str(label.get("name")): label for label in labels}
|
||||||
|
|
||||||
|
|
||||||
|
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||||
|
labels = client.paginate(org_path(owner, "/labels"))
|
||||||
|
return {str(label.get("name")): label for label in labels}
|
||||||
|
|
||||||
|
|
||||||
|
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||||
|
if scope == "organization":
|
||||||
|
return org_path(org, "/labels")
|
||||||
|
return repo_path(owner, repo, "/labels")
|
||||||
|
|
||||||
|
|
||||||
|
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||||
|
if scope == "organization":
|
||||||
|
return org_path(org, f"/labels/{label_id}")
|
||||||
|
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||||
|
|
||||||
|
|
||||||
|
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
patch: dict[str, Any] = {}
|
||||||
|
if _normalize_color(current.get("color")) != desired["color"]:
|
||||||
|
patch["color"] = desired["color"]
|
||||||
|
if str(current.get("description") or "").strip() != desired["description"]:
|
||||||
|
patch["description"] = desired["description"]
|
||||||
|
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
||||||
|
patch["exclusive"] = desired["exclusive"]
|
||||||
|
return patch
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_color(value: Any) -> str:
|
||||||
|
return str(value or "").strip().lstrip("#").lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_hex_color(value: str) -> bool:
|
||||||
|
if len(value) != 6:
|
||||||
|
return False
|
||||||
|
return all(char in "0123456789abcdef" for char in value)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
637
scripts/gitea-sync-wiki.py
Normal file
637
scripts/gitea-sync-wiki.py
Normal file
@@ -0,0 +1,637 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Mirror project documentation files into Gitea repository wikis."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import dataclasses
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||||
|
|
||||||
|
|
||||||
|
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||||
|
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||||
|
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
||||||
|
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
||||||
|
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
||||||
|
EXCLUDED_PARTS = {
|
||||||
|
".git",
|
||||||
|
".gitea",
|
||||||
|
".venv",
|
||||||
|
"node_modules",
|
||||||
|
"__pycache__",
|
||||||
|
"dist",
|
||||||
|
"build",
|
||||||
|
".cache",
|
||||||
|
".module-test-build",
|
||||||
|
}
|
||||||
|
EXCLUDED_NAMES = {
|
||||||
|
"package-lock.json",
|
||||||
|
"pnpm-lock.yaml",
|
||||||
|
"yarn.lock",
|
||||||
|
"uv.lock",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class RepoInfo:
|
||||||
|
root: pathlib.Path
|
||||||
|
owner: str
|
||||||
|
repo: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def key(self) -> str:
|
||||||
|
return self.root.name
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class WikiSource:
|
||||||
|
repo_key: str
|
||||||
|
path: pathlib.Path
|
||||||
|
origin: str
|
||||||
|
page_title: str
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||||
|
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||||
|
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||||
|
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
||||||
|
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
||||||
|
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
||||||
|
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||||
|
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||||
|
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||||
|
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||||
|
parser.add_argument("--apply", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
load_dotenv(args.env_file)
|
||||||
|
repos = discover_hosted_repos(args.git_root)
|
||||||
|
if args.repo:
|
||||||
|
requested = set(args.repo)
|
||||||
|
repos = {key: repo for key, repo in repos.items() if key in requested}
|
||||||
|
missing = requested - set(repos)
|
||||||
|
if missing:
|
||||||
|
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
||||||
|
|
||||||
|
sources = discover_sources(repos, args.products_root)
|
||||||
|
if args.page:
|
||||||
|
requested_pages = set(args.page)
|
||||||
|
sources = [source for source in sources if source.page_title in requested_pages]
|
||||||
|
missing_pages = requested_pages - {source.page_title for source in sources}
|
||||||
|
if missing_pages:
|
||||||
|
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
||||||
|
print(f"Hosted repositories: {len(repos)}")
|
||||||
|
print(f"Wiki source files: {len(sources)}")
|
||||||
|
for repo_key, count in grouped_counts(sources).items():
|
||||||
|
print(f" {repo_key}: {count}")
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
preview_sources(sources)
|
||||||
|
print("Dry run only. Re-run with --apply to write wiki pages.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
token = require_token() if args.transport == "api" else ""
|
||||||
|
failures = 0
|
||||||
|
for repo_key, repo_sources in group_sources(sources).items():
|
||||||
|
repo = repos[repo_key]
|
||||||
|
try:
|
||||||
|
if args.transport == "git":
|
||||||
|
sync_repo_wiki_git(
|
||||||
|
repo,
|
||||||
|
repo_sources,
|
||||||
|
cache_dir=args.wiki_cache_dir,
|
||||||
|
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||||
|
commit_message=args.commit_message,
|
||||||
|
write_index=not bool(args.page),
|
||||||
|
prune_managed=args.prune_managed and not bool(args.page),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sync_repo_wiki(
|
||||||
|
repo,
|
||||||
|
repo_sources,
|
||||||
|
token,
|
||||||
|
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||||
|
write_index=not bool(args.page),
|
||||||
|
prune_managed=args.prune_managed and not bool(args.page),
|
||||||
|
)
|
||||||
|
except GiteaError as exc:
|
||||||
|
failures += 1
|
||||||
|
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||||
|
return 1 if failures else 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||||
|
repos: dict[str, RepoInfo] = {}
|
||||||
|
for gitdir in sorted(git_root.glob("*/.git")):
|
||||||
|
root = gitdir.parent
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if "git.add-ideas.de" not in result.stdout:
|
||||||
|
continue
|
||||||
|
target = infer_target(root)
|
||||||
|
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
||||||
|
return repos
|
||||||
|
|
||||||
|
|
||||||
|
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
||||||
|
sources: list[WikiSource] = []
|
||||||
|
for repo in repos.values():
|
||||||
|
for path in repo.root.rglob("*"):
|
||||||
|
if is_repo_doc(repo.root, path):
|
||||||
|
rel = path.relative_to(repo.root)
|
||||||
|
sources.append(
|
||||||
|
WikiSource(
|
||||||
|
repo_key=repo.key,
|
||||||
|
path=path,
|
||||||
|
origin="repository",
|
||||||
|
page_title=title_for_path("Repo", rel),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
product_map = {
|
||||||
|
"co2api": "emission-api-lib",
|
||||||
|
"govoplan": "govoplan-core",
|
||||||
|
"meubility": "meubility-workbench",
|
||||||
|
"mousehold": "mousehold",
|
||||||
|
"prvnncr": "prvnncr-server",
|
||||||
|
}
|
||||||
|
if products_root.exists():
|
||||||
|
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||||
|
repo_key = product_map.get(product_dir.name)
|
||||||
|
if not repo_key or repo_key not in repos:
|
||||||
|
continue
|
||||||
|
for path in product_dir.rglob("*"):
|
||||||
|
if is_product_doc(path):
|
||||||
|
rel = path.relative_to(product_dir)
|
||||||
|
sources.append(
|
||||||
|
WikiSource(
|
||||||
|
repo_key=repo_key,
|
||||||
|
path=path,
|
||||||
|
origin=f"product:{product_dir.name}",
|
||||||
|
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
unique: dict[tuple[str, str], WikiSource] = {}
|
||||||
|
for source in sources:
|
||||||
|
unique[(source.repo_key, source.page_title)] = source
|
||||||
|
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
||||||
|
|
||||||
|
|
||||||
|
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
||||||
|
if not path.is_file() or not is_text_file(path):
|
||||||
|
return False
|
||||||
|
parts = set(path.relative_to(repo_root_path).parts)
|
||||||
|
if parts & EXCLUDED_PARTS:
|
||||||
|
return False
|
||||||
|
if path.name in EXCLUDED_NAMES:
|
||||||
|
return False
|
||||||
|
if SENSITIVE_NAME_RE.search(path.name):
|
||||||
|
return False
|
||||||
|
rel = path.relative_to(repo_root_path)
|
||||||
|
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
||||||
|
return True
|
||||||
|
if "generated" in rel.parts:
|
||||||
|
return False
|
||||||
|
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
||||||
|
return True
|
||||||
|
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_product_doc(path: pathlib.Path) -> bool:
|
||||||
|
if not path.is_file() or not is_text_file(path):
|
||||||
|
return False
|
||||||
|
if path.name in EXCLUDED_NAMES:
|
||||||
|
return False
|
||||||
|
if SENSITIVE_NAME_RE.search(path.name):
|
||||||
|
return False
|
||||||
|
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
||||||
|
return False
|
||||||
|
if re.search(
|
||||||
|
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
||||||
|
path.name,
|
||||||
|
re.IGNORECASE,
|
||||||
|
) and path.suffix.lower() in {".md", ".txt"}:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def is_text_file(path: pathlib.Path) -> bool:
|
||||||
|
if path.suffix.lower() not in TEXT_SUFFIXES:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
chunk = path.read_bytes()[:4096]
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return b"\x00" not in chunk
|
||||||
|
|
||||||
|
|
||||||
|
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
||||||
|
stem = rel.with_suffix("").as_posix()
|
||||||
|
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
||||||
|
return f"{prefix}-{stem}"[:180]
|
||||||
|
|
||||||
|
|
||||||
|
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
||||||
|
grouped: dict[str, list[WikiSource]] = {}
|
||||||
|
for source in sources:
|
||||||
|
grouped.setdefault(source.repo_key, []).append(source)
|
||||||
|
return dict(sorted(grouped.items()))
|
||||||
|
|
||||||
|
|
||||||
|
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
||||||
|
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
||||||
|
|
||||||
|
|
||||||
|
def preview_sources(sources: list[WikiSource]) -> None:
|
||||||
|
for repo_key, repo_sources in group_sources(sources).items():
|
||||||
|
print(f"{repo_key}:")
|
||||||
|
for source in repo_sources[:80]:
|
||||||
|
print(f" {source.page_title} <- {source.path}")
|
||||||
|
if len(repo_sources) > 80:
|
||||||
|
print(f" ... {len(repo_sources) - 80} more")
|
||||||
|
|
||||||
|
|
||||||
|
def sync_repo_wiki(
|
||||||
|
repo: RepoInfo,
|
||||||
|
sources: list[WikiSource],
|
||||||
|
token: str,
|
||||||
|
*,
|
||||||
|
overwrite_unmanaged: bool,
|
||||||
|
write_index: bool = True,
|
||||||
|
prune_managed: bool = False,
|
||||||
|
) -> None:
|
||||||
|
target = infer_target(repo.root)
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||||
|
existing_page_names = {
|
||||||
|
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
||||||
|
for page in existing_pages
|
||||||
|
if page.get("title")
|
||||||
|
}
|
||||||
|
index_lines = [
|
||||||
|
f"# {target.repo} Project Wiki Index",
|
||||||
|
"",
|
||||||
|
f"{MANAGED_MARKER}index -->",
|
||||||
|
"",
|
||||||
|
"This page is generated from repository and product-directory project files.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
for source in sources:
|
||||||
|
content = render_wiki_content(source)
|
||||||
|
changed = put_wiki_page(
|
||||||
|
client,
|
||||||
|
target.owner,
|
||||||
|
target.repo,
|
||||||
|
source.page_title,
|
||||||
|
content,
|
||||||
|
existing_page_names,
|
||||||
|
overwrite_unmanaged=overwrite_unmanaged,
|
||||||
|
)
|
||||||
|
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||||
|
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||||
|
|
||||||
|
if write_index:
|
||||||
|
put_wiki_page(
|
||||||
|
client,
|
||||||
|
target.owner,
|
||||||
|
target.repo,
|
||||||
|
"Codex-Project-Index",
|
||||||
|
"\n".join(index_lines) + "\n",
|
||||||
|
existing_page_names,
|
||||||
|
overwrite_unmanaged=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||||
|
if prune_managed and write_index:
|
||||||
|
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||||
|
|
||||||
|
|
||||||
|
def sync_repo_wiki_git(
|
||||||
|
repo: RepoInfo,
|
||||||
|
sources: list[WikiSource],
|
||||||
|
*,
|
||||||
|
cache_dir: pathlib.Path,
|
||||||
|
overwrite_unmanaged: bool,
|
||||||
|
commit_message: str,
|
||||||
|
write_index: bool = True,
|
||||||
|
prune_managed: bool = False,
|
||||||
|
) -> None:
|
||||||
|
target = infer_target(repo.root)
|
||||||
|
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||||
|
index_lines = [
|
||||||
|
f"# {target.repo} Project Wiki Index",
|
||||||
|
"",
|
||||||
|
f"{MANAGED_MARKER}index -->",
|
||||||
|
"",
|
||||||
|
"This page is generated from repository and product-directory project files.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
for source in sources:
|
||||||
|
content = render_wiki_content(source)
|
||||||
|
status = write_wiki_page_file(
|
||||||
|
wiki_root,
|
||||||
|
source.page_title,
|
||||||
|
content,
|
||||||
|
overwrite_unmanaged=overwrite_unmanaged,
|
||||||
|
)
|
||||||
|
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||||
|
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||||
|
|
||||||
|
if write_index:
|
||||||
|
write_wiki_page_file(
|
||||||
|
wiki_root,
|
||||||
|
"Codex-Project-Index",
|
||||||
|
"\n".join(index_lines) + "\n",
|
||||||
|
overwrite_unmanaged=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||||
|
if prune_managed and write_index:
|
||||||
|
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||||
|
if not git_has_changes(wiki_root):
|
||||||
|
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||||
|
return
|
||||||
|
run_git(wiki_root, "add", "-A")
|
||||||
|
if not git_has_staged_changes(wiki_root):
|
||||||
|
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||||
|
return
|
||||||
|
ensure_git_identity(wiki_root)
|
||||||
|
run_git(wiki_root, "commit", "-m", commit_message)
|
||||||
|
run_git(wiki_root, "push")
|
||||||
|
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
||||||
|
remote = wiki_remote_for_repo(repo.root)
|
||||||
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
||||||
|
if (checkout / ".git").exists():
|
||||||
|
run_git(checkout, "remote", "set-url", "origin", remote)
|
||||||
|
run_git(checkout, "fetch", "origin")
|
||||||
|
branch = current_branch(checkout)
|
||||||
|
if branch:
|
||||||
|
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
||||||
|
else:
|
||||||
|
run_git(checkout, "reset", "--hard")
|
||||||
|
run_git(checkout, "clean", "-fd")
|
||||||
|
return checkout
|
||||||
|
if checkout.exists():
|
||||||
|
shutil.rmtree(checkout)
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "clone", remote, str(checkout)],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return checkout
|
||||||
|
checkout.mkdir(parents=True, exist_ok=True)
|
||||||
|
run_git(checkout, "init")
|
||||||
|
run_git(checkout, "remote", "add", "origin", remote)
|
||||||
|
run_git(checkout, "checkout", "-b", "master")
|
||||||
|
return checkout
|
||||||
|
|
||||||
|
|
||||||
|
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0 or not result.stdout.strip():
|
||||||
|
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
||||||
|
remote = result.stdout.strip()
|
||||||
|
if remote.endswith(".git"):
|
||||||
|
return f"{remote[:-4]}.wiki.git"
|
||||||
|
return f"{remote}.wiki.git"
|
||||||
|
|
||||||
|
|
||||||
|
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
||||||
|
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
||||||
|
if path.exists():
|
||||||
|
current = read_text(path)
|
||||||
|
if current == content:
|
||||||
|
return "unchanged"
|
||||||
|
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
||||||
|
return "skipped unmanaged"
|
||||||
|
path.write_text(content, encoding="utf-8")
|
||||||
|
return "updated"
|
||||||
|
|
||||||
|
|
||||||
|
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||||
|
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||||
|
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||||
|
for path in sorted(wiki_root.glob("*.md")):
|
||||||
|
if path.name in desired:
|
||||||
|
continue
|
||||||
|
current = read_text(path)
|
||||||
|
if MANAGED_MARKER not in current:
|
||||||
|
continue
|
||||||
|
path.unlink()
|
||||||
|
print(f"deleted stale managed wiki page {path.name}")
|
||||||
|
|
||||||
|
|
||||||
|
def prune_managed_wiki_pages_api(
|
||||||
|
client: GiteaClient,
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
existing_page_names: dict[str, str],
|
||||||
|
sources: list[WikiSource],
|
||||||
|
) -> None:
|
||||||
|
desired = {source.page_title for source in sources}
|
||||||
|
desired.add("Codex-Project-Index")
|
||||||
|
for title, page_name in sorted(existing_page_names.items()):
|
||||||
|
if title in desired:
|
||||||
|
continue
|
||||||
|
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||||
|
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||||
|
continue
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||||
|
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||||
|
|
||||||
|
|
||||||
|
def wiki_file_stem(title: str) -> str:
|
||||||
|
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||||
|
|
||||||
|
|
||||||
|
def git_has_changes(root: pathlib.Path) -> bool:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "status", "--porcelain"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
||||||
|
return bool(result.stdout.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return False
|
||||||
|
if result.returncode == 1:
|
||||||
|
return True
|
||||||
|
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def current_branch(root: pathlib.Path) -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
branch = result.stdout.strip()
|
||||||
|
if result.returncode != 0 or branch == "HEAD":
|
||||||
|
return ""
|
||||||
|
return branch
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_git_identity(root: pathlib.Path) -> None:
|
||||||
|
if not git_config_value(root, "user.email"):
|
||||||
|
run_git(root, "config", "user.email", "codex@govoplan.local")
|
||||||
|
if not git_config_value(root, "user.name"):
|
||||||
|
run_git(root, "config", "user.name", "Codex")
|
||||||
|
|
||||||
|
|
||||||
|
def git_config_value(root: pathlib.Path, key: str) -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "config", "--get", key],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
return result.stdout.strip() if result.returncode == 0 else ""
|
||||||
|
|
||||||
|
|
||||||
|
def run_git(root: pathlib.Path, *args: str) -> None:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), *args],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
command = "git -C " + str(root) + " " + " ".join(args)
|
||||||
|
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||||
|
try:
|
||||||
|
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
||||||
|
except GiteaError as exc:
|
||||||
|
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
||||||
|
return []
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def render_wiki_content(source: WikiSource) -> str:
|
||||||
|
raw = read_text(source.path)
|
||||||
|
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||||
|
header = [
|
||||||
|
f"{MANAGED_MARKER}{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
f"> Mirrored from `{source.path}`.",
|
||||||
|
f"> Origin: `{source.origin}`.",
|
||||||
|
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
||||||
|
"",
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
if source.path.suffix.lower() == ".csv":
|
||||||
|
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
||||||
|
return "\n".join(header) + raw.rstrip() + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def put_wiki_page(
|
||||||
|
client: GiteaClient,
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
title: str,
|
||||||
|
content: str,
|
||||||
|
existing_page_names: dict[str, str],
|
||||||
|
*,
|
||||||
|
overwrite_unmanaged: bool,
|
||||||
|
) -> bool:
|
||||||
|
body = {
|
||||||
|
"title": title,
|
||||||
|
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
||||||
|
"message": f"Sync {title} from project files",
|
||||||
|
}
|
||||||
|
if title in existing_page_names:
|
||||||
|
page_name = existing_page_names[title]
|
||||||
|
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||||
|
current_content = decode_wiki_content(current)
|
||||||
|
if current_content == content:
|
||||||
|
return False
|
||||||
|
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
||||||
|
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
||||||
|
return False
|
||||||
|
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
||||||
|
return True
|
||||||
|
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
||||||
|
existing_page_names[title] = title
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def quote_wiki_page_name(value: str) -> str:
|
||||||
|
return urllib.parse.quote(value, safe="+")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_wiki_content(page: dict[str, Any]) -> str:
|
||||||
|
encoded = str(page.get("content_base64") or "")
|
||||||
|
if not encoded:
|
||||||
|
return ""
|
||||||
|
return base64.b64decode(encoded).decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(path: pathlib.Path) -> str:
|
||||||
|
try:
|
||||||
|
return path.read_text(encoding="utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return path.read_text(encoding="latin-1")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
377
scripts/gitea-todo-import.py
Normal file
377
scripts/gitea-todo-import.py
Normal file
@@ -0,0 +1,377 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Preview or import inline TODO-style markers as Gitea issues."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||||
|
|
||||||
|
|
||||||
|
MARKER_RE = re.compile(
|
||||||
|
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
||||||
|
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
||||||
|
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
||||||
|
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
DEFAULT_EXCLUDES = (
|
||||||
|
"!**/.git/**",
|
||||||
|
"!**/.venv/**",
|
||||||
|
"!**/node_modules/**",
|
||||||
|
"!**/__pycache__/**",
|
||||||
|
"!**/.module-test-build/**",
|
||||||
|
"!**/dist/**",
|
||||||
|
"!**/build/**",
|
||||||
|
"!**/.cache/**",
|
||||||
|
"!.gitea/**",
|
||||||
|
"!docs/GITEA_ISSUES.md",
|
||||||
|
"!scripts/gitea-todo-import.py",
|
||||||
|
"!scripts/gitea-sync-labels.py",
|
||||||
|
"!scripts/gitea-codex-note.py",
|
||||||
|
"!scripts/gitea_common.py",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||||
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||||
|
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
||||||
|
parser.add_argument(
|
||||||
|
"--module-label",
|
||||||
|
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
||||||
|
)
|
||||||
|
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
||||||
|
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
||||||
|
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
||||||
|
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = repo_root(args.root)
|
||||||
|
load_dotenv(args.env_file or root / ".env")
|
||||||
|
target = infer_target(root, args.remote)
|
||||||
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
|
||||||
|
markers = scan_markers(root, include_linked=args.include_linked)
|
||||||
|
if args.limit > 0:
|
||||||
|
markers = markers[: args.limit]
|
||||||
|
|
||||||
|
module_label = args.module_label or infer_module_label(root)
|
||||||
|
previews = [
|
||||||
|
build_issue_preview(
|
||||||
|
target.owner,
|
||||||
|
target.repo,
|
||||||
|
marker,
|
||||||
|
module_label=module_label,
|
||||||
|
infer_areas=not args.no_area_labels,
|
||||||
|
extra_labels=args.extra_label,
|
||||||
|
)
|
||||||
|
for marker in markers
|
||||||
|
]
|
||||||
|
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
||||||
|
else:
|
||||||
|
print(f"Target: {target.display}")
|
||||||
|
print(f"Scanned root: {root}")
|
||||||
|
print(f"Found {len(previews)} importable marker(s).")
|
||||||
|
for preview in previews:
|
||||||
|
print(f"- {preview.title}")
|
||||||
|
print(f" {preview.location}")
|
||||||
|
print(f" labels: {', '.join(preview.labels)}")
|
||||||
|
|
||||||
|
if not args.apply:
|
||||||
|
if not previews:
|
||||||
|
return 0
|
||||||
|
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if not previews:
|
||||||
|
print("No issues to create.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
client = GiteaClient(target, token)
|
||||||
|
label_ids = load_label_ids(client, target.owner, target.repo)
|
||||||
|
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
||||||
|
if missing_labels:
|
||||||
|
joined = ", ".join(missing_labels)
|
||||||
|
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
||||||
|
|
||||||
|
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
||||||
|
created = 0
|
||||||
|
skipped = 0
|
||||||
|
for preview in previews:
|
||||||
|
if preview.fingerprint in existing_fingerprints:
|
||||||
|
print(f"skip existing {preview.location}")
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
issue = client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(target.owner, target.repo, "/issues"),
|
||||||
|
body={
|
||||||
|
"title": preview.title,
|
||||||
|
"body": preview.body,
|
||||||
|
"labels": [label_ids[label] for label in preview.labels],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
||||||
|
created += 1
|
||||||
|
|
||||||
|
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
class Marker:
|
||||||
|
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
||||||
|
self.path = path
|
||||||
|
self.line = line
|
||||||
|
self.column = column
|
||||||
|
self.marker = marker.upper()
|
||||||
|
self.context = context
|
||||||
|
self.text = text
|
||||||
|
self.raw = raw.rstrip("\n")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def location(self) -> str:
|
||||||
|
return f"{self.path}:{self.line}"
|
||||||
|
|
||||||
|
|
||||||
|
class IssuePreview:
|
||||||
|
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
||||||
|
self.title = title
|
||||||
|
self.body = body
|
||||||
|
self.labels = labels
|
||||||
|
self.fingerprint = fingerprint
|
||||||
|
self.location = location
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"title": self.title,
|
||||||
|
"body": self.body,
|
||||||
|
"labels": self.labels,
|
||||||
|
"fingerprint": self.fingerprint,
|
||||||
|
"location": self.location,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
||||||
|
command = [
|
||||||
|
"rg",
|
||||||
|
"--json",
|
||||||
|
"--hidden",
|
||||||
|
"--line-number",
|
||||||
|
"--column",
|
||||||
|
"-e",
|
||||||
|
r"\b(TODO|FIXME|XXX|HACK)\b",
|
||||||
|
]
|
||||||
|
for pattern in DEFAULT_EXCLUDES:
|
||||||
|
command.extend(["--glob", pattern])
|
||||||
|
command.append(str(root))
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
command,
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode not in {0, 1}:
|
||||||
|
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
||||||
|
|
||||||
|
markers: list[Marker] = []
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
event = json.loads(line)
|
||||||
|
if event.get("type") != "match":
|
||||||
|
continue
|
||||||
|
data = event["data"]
|
||||||
|
raw_line = data["lines"]["text"]
|
||||||
|
match = MARKER_RE.search(raw_line)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
if not _looks_like_marker(match):
|
||||||
|
continue
|
||||||
|
context = (match.group("context") or "").strip()
|
||||||
|
if not include_linked and _already_linked(context, raw_line):
|
||||||
|
continue
|
||||||
|
markers.append(
|
||||||
|
Marker(
|
||||||
|
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
||||||
|
line=int(data["line_number"]),
|
||||||
|
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
||||||
|
marker=match.group("marker"),
|
||||||
|
context=context,
|
||||||
|
text=clean_marker_text(match.group("text") or raw_line),
|
||||||
|
raw=raw_line,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return markers
|
||||||
|
|
||||||
|
|
||||||
|
def build_issue_preview(
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
marker: Marker,
|
||||||
|
*,
|
||||||
|
module_label: str,
|
||||||
|
infer_areas: bool,
|
||||||
|
extra_labels: list[str],
|
||||||
|
) -> IssuePreview:
|
||||||
|
fingerprint = marker_fingerprint(owner, repo, marker)
|
||||||
|
summary = marker.text or f"review {marker.location}"
|
||||||
|
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
||||||
|
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
||||||
|
body = "\n".join(
|
||||||
|
[
|
||||||
|
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
||||||
|
"",
|
||||||
|
"Imported from an inline marker.",
|
||||||
|
"",
|
||||||
|
f"- Repository: `{owner}/{repo}`",
|
||||||
|
f"- Location: `{marker.location}`",
|
||||||
|
f"- Marker: `{marker.marker}`",
|
||||||
|
f"- Context: `{marker.context or 'none'}`",
|
||||||
|
"",
|
||||||
|
"Current line:",
|
||||||
|
"",
|
||||||
|
"```text",
|
||||||
|
marker.raw,
|
||||||
|
"```",
|
||||||
|
"",
|
||||||
|
"Migration rule:",
|
||||||
|
"",
|
||||||
|
"- Keep this Gitea issue as the canonical backlog item.",
|
||||||
|
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_marker_text(text: str) -> str:
|
||||||
|
cleaned = text.strip()
|
||||||
|
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
||||||
|
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
||||||
|
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
||||||
|
area = infer_area_label(marker.path) if infer_areas else ""
|
||||||
|
if area:
|
||||||
|
labels.append(area)
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def marker_type_label(marker: str) -> str:
|
||||||
|
if marker == "FIXME":
|
||||||
|
return "type/bug"
|
||||||
|
if marker in {"HACK", "XXX"}:
|
||||||
|
return "type/debt"
|
||||||
|
return "type/task"
|
||||||
|
|
||||||
|
|
||||||
|
def infer_area_label(path: pathlib.Path) -> str:
|
||||||
|
text = path.as_posix()
|
||||||
|
if text.startswith("webui/"):
|
||||||
|
return "area/webui"
|
||||||
|
if text.startswith("alembic/"):
|
||||||
|
return "area/migrations"
|
||||||
|
if text.startswith("docs/"):
|
||||||
|
return "area/docs"
|
||||||
|
if text.startswith("scripts/"):
|
||||||
|
return "area/devex"
|
||||||
|
if "/rbac" in text or "permission" in text:
|
||||||
|
return "area/rbac"
|
||||||
|
if "/access" in text or "auth" in text or "session" in text:
|
||||||
|
return "area/auth"
|
||||||
|
if "tenant" in text:
|
||||||
|
return "area/tenancy"
|
||||||
|
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
||||||
|
return "area/governance"
|
||||||
|
if "module" in text:
|
||||||
|
return "area/module-system"
|
||||||
|
if "migration" in text:
|
||||||
|
return "area/migrations"
|
||||||
|
if "router" in text or "api" in text:
|
||||||
|
return "area/api"
|
||||||
|
if "db" in text or "model" in text or "persistence" in text:
|
||||||
|
return "area/db"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def infer_module_label(root: pathlib.Path) -> str:
|
||||||
|
known = {
|
||||||
|
"govoplan-core": "module/core",
|
||||||
|
"govoplan-access": "module/access",
|
||||||
|
"govoplan-admin": "module/admin",
|
||||||
|
"govoplan-tenancy": "module/tenancy",
|
||||||
|
"govoplan-policy": "module/policy",
|
||||||
|
"govoplan-audit": "module/audit",
|
||||||
|
"govoplan-mail": "module/mail",
|
||||||
|
"govoplan-files": "module/files",
|
||||||
|
"govoplan-campaign": "module/campaign",
|
||||||
|
"govoplan-web": "module/web",
|
||||||
|
}
|
||||||
|
if root.name in known:
|
||||||
|
return known[root.name]
|
||||||
|
if root.name.startswith("govoplan-"):
|
||||||
|
suffix = root.name.removeprefix("govoplan-")
|
||||||
|
if suffix:
|
||||||
|
return f"module/{suffix}"
|
||||||
|
return "module/core"
|
||||||
|
|
||||||
|
|
||||||
|
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
||||||
|
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
||||||
|
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
def truncate_title(title: str, limit: int = 180) -> str:
|
||||||
|
if len(title) <= limit:
|
||||||
|
return title
|
||||||
|
return f"{title[: limit - 1].rstrip()}..."
|
||||||
|
|
||||||
|
|
||||||
|
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||||
|
return label_ids_by_name(client, owner, repo)
|
||||||
|
|
||||||
|
|
||||||
|
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||||
|
issues = client.paginate(
|
||||||
|
repo_path(owner, repo, "/issues"),
|
||||||
|
query={"state": "all", "labels": "source/todo-scan"},
|
||||||
|
)
|
||||||
|
fingerprints: set[str] = set()
|
||||||
|
for issue in issues:
|
||||||
|
body = str(issue.get("body") or "")
|
||||||
|
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
||||||
|
fingerprints.add(match.group(1))
|
||||||
|
return fingerprints
|
||||||
|
|
||||||
|
|
||||||
|
def _already_linked(context: str, raw_line: str) -> bool:
|
||||||
|
linked_text = f"{context} {raw_line}".lower()
|
||||||
|
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_like_marker(match: re.Match[str]) -> bool:
|
||||||
|
text = (match.group("text") or "").strip()
|
||||||
|
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
290
scripts/gitea_common.py
Normal file
290
scripts/gitea_common.py
Normal file
@@ -0,0 +1,290 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Shared helpers for Gitea maintenance scripts."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class GiteaError(RuntimeError):
|
||||||
|
"""Raised when a Gitea API request fails."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass(frozen=True)
|
||||||
|
class RepoTarget:
|
||||||
|
base_url: str
|
||||||
|
owner: str
|
||||||
|
repo: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def display(self) -> str:
|
||||||
|
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api_base(self) -> str:
|
||||||
|
return f"{self.base_url.rstrip('/')}/api/v1"
|
||||||
|
|
||||||
|
|
||||||
|
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return start.resolve()
|
||||||
|
return pathlib.Path(result.stdout.strip()).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
||||||
|
env_url = os.environ.get("GITEA_URL")
|
||||||
|
env_owner = os.environ.get("GITEA_OWNER")
|
||||||
|
env_repo = os.environ.get("GITEA_REPO")
|
||||||
|
|
||||||
|
remote_url = _git_remote(root, remote_name)
|
||||||
|
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
||||||
|
|
||||||
|
base_url = (env_url or inferred_url).rstrip("/")
|
||||||
|
owner = env_owner or inferred_owner
|
||||||
|
repo = env_repo or inferred_repo
|
||||||
|
|
||||||
|
missing = [
|
||||||
|
name
|
||||||
|
for name, value in (
|
||||||
|
("GITEA_URL", base_url),
|
||||||
|
("GITEA_OWNER", owner),
|
||||||
|
("GITEA_REPO", repo),
|
||||||
|
)
|
||||||
|
if not value
|
||||||
|
]
|
||||||
|
if missing:
|
||||||
|
joined = ", ".join(missing)
|
||||||
|
raise GiteaError(
|
||||||
|
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
||||||
|
)
|
||||||
|
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
||||||
|
|
||||||
|
|
||||||
|
def quote_path(value: str) -> str:
|
||||||
|
return urllib.parse.quote(value, safe="")
|
||||||
|
|
||||||
|
|
||||||
|
class GiteaClient:
|
||||||
|
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
||||||
|
self.target = target
|
||||||
|
self.token = token
|
||||||
|
|
||||||
|
def request_json(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
*,
|
||||||
|
body: dict[str, Any] | None = None,
|
||||||
|
query: dict[str, Any] | None = None,
|
||||||
|
) -> Any:
|
||||||
|
url = f"{self.target.api_base}{path}"
|
||||||
|
if query:
|
||||||
|
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
||||||
|
|
||||||
|
data = None
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json",
|
||||||
|
"User-Agent": "codex-gitea-maintenance",
|
||||||
|
}
|
||||||
|
if self.token:
|
||||||
|
headers["Authorization"] = f"token {self.token}"
|
||||||
|
if body is not None:
|
||||||
|
data = json.dumps(body).encode("utf-8")
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
|
||||||
|
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=30) as response:
|
||||||
|
payload = response.read().decode("utf-8")
|
||||||
|
if not payload:
|
||||||
|
return None
|
||||||
|
return json.loads(payload)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
message = exc.read().decode("utf-8", errors="replace")
|
||||||
|
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
||||||
|
except urllib.error.URLError as exc:
|
||||||
|
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
||||||
|
|
||||||
|
def paginate(
|
||||||
|
self,
|
||||||
|
path: str,
|
||||||
|
*,
|
||||||
|
query: dict[str, Any] | None = None,
|
||||||
|
limit: int = 100,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
page = 1
|
||||||
|
effective_limit = min(limit, 50)
|
||||||
|
seen_pages: set[tuple[Any, ...]] = set()
|
||||||
|
while True:
|
||||||
|
page_query = dict(query or {})
|
||||||
|
page_query.update({"page": page, "limit": effective_limit})
|
||||||
|
payload = self.request_json("GET", path, query=page_query)
|
||||||
|
if not isinstance(payload, list):
|
||||||
|
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
||||||
|
if not payload:
|
||||||
|
return items
|
||||||
|
signature = tuple(
|
||||||
|
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
||||||
|
for item in payload
|
||||||
|
if isinstance(item, dict)
|
||||||
|
)
|
||||||
|
if signature in seen_pages:
|
||||||
|
return items
|
||||||
|
seen_pages.add(signature)
|
||||||
|
items.extend(payload)
|
||||||
|
if len(payload) < effective_limit:
|
||||||
|
return items
|
||||||
|
page += 1
|
||||||
|
|
||||||
|
|
||||||
|
def load_json(path: pathlib.Path) -> Any:
|
||||||
|
with path.open("r", encoding="utf-8") as handle:
|
||||||
|
return json.load(handle)
|
||||||
|
|
||||||
|
|
||||||
|
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
||||||
|
if path is None or not path.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
loaded: list[str] = []
|
||||||
|
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped or stripped.startswith("#"):
|
||||||
|
continue
|
||||||
|
if stripped.startswith("export "):
|
||||||
|
stripped = stripped[len("export ") :].strip()
|
||||||
|
if "=" not in stripped:
|
||||||
|
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
||||||
|
key, value = stripped.split("=", 1)
|
||||||
|
key = key.strip()
|
||||||
|
value = _strip_env_value(value.strip())
|
||||||
|
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||||||
|
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
||||||
|
if key not in os.environ:
|
||||||
|
os.environ[key] = value
|
||||||
|
loaded.append(key)
|
||||||
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
|
def require_token() -> str:
|
||||||
|
token = os.environ.get("GITEA_TOKEN")
|
||||||
|
if not token:
|
||||||
|
raise GiteaError(
|
||||||
|
"GITEA_TOKEN is required when using --apply. "
|
||||||
|
"Set it in the environment, the target .env, or --env-file."
|
||||||
|
)
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||||
|
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def org_path(owner: str, suffix: str) -> str:
|
||||||
|
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||||
|
"""Return org labels plus optional repository labels, keyed by label name.
|
||||||
|
|
||||||
|
Gitea stores organization labels separately from repository labels. Issue
|
||||||
|
APIs can use label ids from both scopes on supported instances, so issue
|
||||||
|
creation helpers should resolve both. Repository labels intentionally win
|
||||||
|
when a repo carries a local label with the same name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
labels: dict[str, dict[str, Any]] = {}
|
||||||
|
try:
|
||||||
|
for label in client.paginate(org_path(owner, "/labels")):
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
if name:
|
||||||
|
labels[name] = label
|
||||||
|
except GiteaError:
|
||||||
|
# User-owned repositories or older instances may not expose org labels.
|
||||||
|
pass
|
||||||
|
|
||||||
|
if repo:
|
||||||
|
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
if name:
|
||||||
|
labels[name] = label
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||||
|
ids: dict[str, int] = {}
|
||||||
|
for name, label in labels_by_name(client, owner, repo).items():
|
||||||
|
label_id = label.get("id") or label.get("index")
|
||||||
|
if label_id is None:
|
||||||
|
continue
|
||||||
|
ids[name] = int(label_id)
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||||
|
check=False,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return ""
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
||||||
|
if not remote_url:
|
||||||
|
return "", "", ""
|
||||||
|
|
||||||
|
parsed = urllib.parse.urlparse(remote_url)
|
||||||
|
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
||||||
|
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||||
|
owner, repo = _owner_repo_from_parts(path_parts)
|
||||||
|
if parsed.scheme in {"http", "https"}:
|
||||||
|
prefix = "/".join(path_parts[:-2])
|
||||||
|
base_path = f"/{prefix}" if prefix else ""
|
||||||
|
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
||||||
|
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
||||||
|
|
||||||
|
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
||||||
|
if scp_like:
|
||||||
|
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
||||||
|
owner, repo = _owner_repo_from_parts(path_parts)
|
||||||
|
return f"https://{scp_like.group('host')}", owner, repo
|
||||||
|
|
||||||
|
return "", "", ""
|
||||||
|
|
||||||
|
|
||||||
|
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
||||||
|
if len(path_parts) < 2:
|
||||||
|
return "", ""
|
||||||
|
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_git_suffix(value: str) -> str:
|
||||||
|
return value[:-4] if value.endswith(".git") else value
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_env_value(value: str) -> str:
|
||||||
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||||
|
return value[1:-1]
|
||||||
|
return value
|
||||||
74
scripts/install-webui-release-dependencies.sh
Normal file
74
scripts/install-webui-release-dependencies.sh
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
WEBUI_DIR="${1:-$ROOT/webui}"
|
||||||
|
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
|
||||||
|
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
|
||||||
|
export GOVOPLAN_DEPS
|
||||||
|
|
||||||
|
trap 'rm -rf "$WORK_ROOT"' EXIT
|
||||||
|
|
||||||
|
retry() {
|
||||||
|
local attempt status
|
||||||
|
for attempt in 1 2 3; do
|
||||||
|
if "$@"; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
status=$?
|
||||||
|
if [[ "$attempt" == 3 ]]; then
|
||||||
|
return "$status"
|
||||||
|
fi
|
||||||
|
sleep $((attempt * 10))
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cd "$WEBUI_DIR"
|
||||||
|
|
||||||
|
node <<'JS'
|
||||||
|
const fs = require("fs");
|
||||||
|
|
||||||
|
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||||
|
const rel = JSON.parse(fs.readFileSync("package.release.json", "utf8"));
|
||||||
|
const govoplanDeps = [];
|
||||||
|
const dependencies = {...(rel.dependencies || {})};
|
||||||
|
|
||||||
|
for (const [name, spec] of Object.entries(dependencies)) {
|
||||||
|
if (name.startsWith("@govoplan/")) {
|
||||||
|
govoplanDeps.push([name, spec]);
|
||||||
|
delete dependencies[name];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of ["devDependencies", "peerDependencies", "optionalDependencies", "overrides"]) {
|
||||||
|
if (rel[key]) pkg[key] = rel[key];
|
||||||
|
}
|
||||||
|
pkg.dependencies = dependencies;
|
||||||
|
|
||||||
|
fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
|
||||||
|
fs.writeFileSync(process.env.GOVOPLAN_DEPS, govoplanDeps.map((item) => item.join("\t")).join("\n") + "\n");
|
||||||
|
JS
|
||||||
|
|
||||||
|
rm -f package-lock.json
|
||||||
|
npm cache clean --force
|
||||||
|
retry npm install --prefer-online
|
||||||
|
|
||||||
|
module_paths=()
|
||||||
|
while IFS=$'\t' read -r package_name spec; do
|
||||||
|
[[ -n "${package_name:-}" ]] || continue
|
||||||
|
git_url="${spec%%#*}"
|
||||||
|
git_ref="${spec#*#}"
|
||||||
|
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
|
||||||
|
echo "Unsupported GovOPlaN WebUI git spec for $package_name: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
clone_url="${git_url#git+}"
|
||||||
|
clone_dir="$WORK_ROOT/${package_name#@govoplan/}"
|
||||||
|
echo "Installing $package_name from $clone_url#$git_ref"
|
||||||
|
retry git clone --depth 1 --branch "$git_ref" "$clone_url" "$clone_dir"
|
||||||
|
module_paths+=("file:$clone_dir")
|
||||||
|
done < "$GOVOPLAN_DEPS"
|
||||||
|
|
||||||
|
if [[ "${#module_paths[@]}" -gt 0 ]]; then
|
||||||
|
retry npm install --prefer-online --no-save --install-links "${module_paths[@]}"
|
||||||
|
fi
|
||||||
171
scripts/launch-dev.sh
Normal file
171
scripts/launch-dev.sh
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||||
|
WEBUI_ROOT="$ROOT/webui"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||||
|
NPM="${NPM:-$NODE_BIN/npm}"
|
||||||
|
|
||||||
|
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||||
|
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||||
|
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||||
|
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||||
|
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||||
|
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
||||||
|
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||||
|
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||||
|
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||||
|
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
||||||
|
|
||||||
|
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||||
|
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||||
|
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||||
|
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||||
|
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||||
|
|
||||||
|
backend_pid=""
|
||||||
|
frontend_pid=""
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
printf 'launch-dev: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$DEV_DATABASE_BACKEND" in
|
||||||
|
postgres)
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||||
|
;;
|
||||||
|
sqlite)
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||||
|
|
||||||
|
port_is_free() {
|
||||||
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
host = sys.argv[1]
|
||||||
|
port = int(sys.argv[2])
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
try:
|
||||||
|
sock.bind((host, port))
|
||||||
|
except OSError:
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_url() {
|
||||||
|
"$PYTHON" - "$1" <<'PY'
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
url = sys.argv[1]
|
||||||
|
deadline = time.monotonic() + 60
|
||||||
|
last_error = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=2) as response:
|
||||||
|
if 200 <= response.status < 500:
|
||||||
|
raise SystemExit(0)
|
||||||
|
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||||
|
kill "$frontend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||||
|
kill "$backend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||||
|
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||||
|
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||||
|
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||||
|
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
: > "$BACKEND_LOG"
|
||||||
|
: > "$FRONTEND_LOG"
|
||||||
|
|
||||||
|
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
|
||||||
|
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
|
||||||
|
fi
|
||||||
|
|
||||||
|
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||||
|
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||||
|
) >"$BACKEND_LOG" 2>&1 &
|
||||||
|
backend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||||
|
wait_for_url "$BACKEND_URL/health" || {
|
||||||
|
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||||
|
fail "backend did not become healthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||||
|
(
|
||||||
|
cd "$WEBUI_ROOT"
|
||||||
|
frontend_args=(run dev)
|
||||||
|
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
|
||||||
|
frontend_args+=(-- --force)
|
||||||
|
fi
|
||||||
|
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||||
|
CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \
|
||||||
|
CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \
|
||||||
|
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||||
|
"$NPM" "${frontend_args[@]}"
|
||||||
|
) >"$FRONTEND_LOG" 2>&1 &
|
||||||
|
frontend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||||
|
wait_for_url "$FRONTEND_URL" || {
|
||||||
|
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||||
|
fail "frontend did not become reachable"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||||
|
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat <<EOF
|
||||||
|
|
||||||
|
GovOPlaN is running.
|
||||||
|
Web UI: $FRONTEND_URL
|
||||||
|
API: $BACKEND_URL/api/v1
|
||||||
|
Health: $BACKEND_URL/health
|
||||||
|
DB: $DATABASE_URL
|
||||||
|
|
||||||
|
Logs:
|
||||||
|
Backend: $BACKEND_LOG
|
||||||
|
WebUI: $FRONTEND_LOG
|
||||||
|
|
||||||
|
Frontend reload:
|
||||||
|
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
|
||||||
|
Vite --force: $FRONTEND_FORCE_RELOAD
|
||||||
|
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
|
||||||
|
|
||||||
|
Press Ctrl+C to stop both processes.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
wait
|
||||||
241
scripts/launch-production-like-dev.sh
Normal file
241
scripts/launch-production-like-dev.sh
Normal file
@@ -0,0 +1,241 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||||
|
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||||
|
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||||
|
if [ ! -f "$PROFILE_ENV" ]; then
|
||||||
|
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||||
|
fi
|
||||||
|
|
||||||
|
WEBUI_ROOT="$ROOT/webui"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||||
|
NPM="${NPM:-$NODE_BIN/npm}"
|
||||||
|
DOCKER="${DOCKER:-docker}"
|
||||||
|
|
||||||
|
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||||
|
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||||
|
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||||
|
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||||
|
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||||
|
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
||||||
|
|
||||||
|
LOG_DIR="$ROOT/runtime/production-like/logs"
|
||||||
|
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||||
|
WORKER_LOG="$LOG_DIR/worker.log"
|
||||||
|
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||||
|
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||||
|
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||||
|
|
||||||
|
backend_pid=""
|
||||||
|
worker_pid=""
|
||||||
|
frontend_pid=""
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$PROFILE_ENV"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
export APP_ENV="${APP_ENV:-staging}"
|
||||||
|
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||||
|
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||||
|
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||||
|
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
||||||
|
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||||
|
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||||
|
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||||
|
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||||
|
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
||||||
|
|
||||||
|
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||||
|
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
||||||
|
from cryptography.fernet import Fernet
|
||||||
|
print(Fernet.generate_key().decode())
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
export MASTER_KEY_B64
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||||
|
|
||||||
|
compose() {
|
||||||
|
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
port_is_free() {
|
||||||
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
host = sys.argv[1]
|
||||||
|
port = int(sys.argv[2])
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
try:
|
||||||
|
sock.bind((host, port))
|
||||||
|
except OSError:
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_tcp() {
|
||||||
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
host = sys.argv[1]
|
||||||
|
port = int(sys.argv[2])
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
last_error = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with socket.create_connection((host, port), timeout=2):
|
||||||
|
raise SystemExit(0)
|
||||||
|
except OSError as exc:
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_url() {
|
||||||
|
"$PYTHON" - "$1" <<'PY'
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
url = sys.argv[1]
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
last_error = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=2) as response:
|
||||||
|
if 200 <= response.status < 500:
|
||||||
|
raise SystemExit(0)
|
||||||
|
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||||
|
kill "$frontend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||||
|
kill "$backend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
||||||
|
kill "$worker_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
||||||
|
compose down >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||||
|
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||||
|
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||||
|
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||||
|
|
||||||
|
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
||||||
|
: > "$BACKEND_LOG"
|
||||||
|
: > "$WORKER_LOG"
|
||||||
|
: > "$FRONTEND_LOG"
|
||||||
|
|
||||||
|
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||||
|
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||||
|
|
||||||
|
printf 'Starting production-like dependencies through Docker Compose\n'
|
||||||
|
compose up -d
|
||||||
|
|
||||||
|
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
||||||
|
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
||||||
|
|
||||||
|
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||||
|
)
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||||
|
) >"$BACKEND_LOG" 2>&1 &
|
||||||
|
backend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||||
|
wait_for_url "$BACKEND_URL/health" || {
|
||||||
|
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||||
|
fail "backend did not become healthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
||||||
|
--queues "$CELERY_QUEUES" \
|
||||||
|
--hostname "govoplan-production-like@%h" \
|
||||||
|
--loglevel INFO
|
||||||
|
) >"$WORKER_LOG" 2>&1 &
|
||||||
|
worker_pid="$!"
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||||
|
(
|
||||||
|
cd "$WEBUI_ROOT"
|
||||||
|
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||||
|
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
||||||
|
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
||||||
|
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||||
|
"$NPM" run dev -- --force
|
||||||
|
) >"$FRONTEND_LOG" 2>&1 &
|
||||||
|
frontend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||||
|
wait_for_url "$FRONTEND_URL" || {
|
||||||
|
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||||
|
fail "frontend did not become reachable"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||||
|
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat <<EOF
|
||||||
|
|
||||||
|
GovOPlaN production-like development profile is running.
|
||||||
|
Web UI: $FRONTEND_URL
|
||||||
|
API: $BACKEND_URL/api/v1
|
||||||
|
Health: $BACKEND_URL/health
|
||||||
|
Ops: $BACKEND_URL/api/v1/ops/status
|
||||||
|
DB: $DATABASE_URL
|
||||||
|
Redis: $REDIS_URL
|
||||||
|
Files: $FILE_STORAGE_LOCAL_ROOT
|
||||||
|
|
||||||
|
Logs:
|
||||||
|
Backend: $BACKEND_LOG
|
||||||
|
Worker: $WORKER_LOG
|
||||||
|
WebUI: $FRONTEND_LOG
|
||||||
|
|
||||||
|
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
||||||
|
Press Ctrl+C to stop API, worker, and WebUI.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
wait
|
||||||
512
scripts/module-installer-rollback-drill.py
Normal file
512
scripts/module-installer-rollback-drill.py
Normal file
@@ -0,0 +1,512 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Callable, Iterable
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SRC = ROOT / "src"
|
||||||
|
if str(SRC) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SRC))
|
||||||
|
|
||||||
|
os.environ.setdefault("APP_ENV", "test")
|
||||||
|
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
||||||
|
os.environ.setdefault("CELERY_ENABLED", "false")
|
||||||
|
|
||||||
|
from govoplan_core.admin.models import SystemSettings
|
||||||
|
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
||||||
|
from govoplan_core.core.module_installer import (
|
||||||
|
cancel_module_installer_request,
|
||||||
|
claim_next_module_installer_request,
|
||||||
|
module_installer_daemon_status,
|
||||||
|
module_installer_lock_status,
|
||||||
|
queue_module_installer_request,
|
||||||
|
read_module_installer_run,
|
||||||
|
retry_module_installer_request,
|
||||||
|
rollback_module_install_run,
|
||||||
|
run_module_install_plan,
|
||||||
|
supervise_module_install_plan,
|
||||||
|
update_module_installer_daemon_status,
|
||||||
|
update_module_installer_request,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.module_management import (
|
||||||
|
ModuleInstallPlan,
|
||||||
|
ModuleInstallPlanItem,
|
||||||
|
saved_desired_enabled_modules,
|
||||||
|
saved_module_install_plan,
|
||||||
|
save_desired_enabled_modules,
|
||||||
|
save_module_install_plan,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.db.session import configure_database, get_database
|
||||||
|
from govoplan_core.server.registry import available_module_manifests
|
||||||
|
|
||||||
|
|
||||||
|
Scenario = Callable[[Path], dict[str, object]]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
||||||
|
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
||||||
|
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
||||||
|
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
||||||
|
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
||||||
|
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
selected = args.scenario or sorted(SCENARIOS)
|
||||||
|
results: list[dict[str, object]] = []
|
||||||
|
ok = True
|
||||||
|
try:
|
||||||
|
for name in selected:
|
||||||
|
scenario_root = runtime_root / name
|
||||||
|
scenario_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
try:
|
||||||
|
result = SCENARIOS[name](scenario_root)
|
||||||
|
except Exception as exc:
|
||||||
|
ok = False
|
||||||
|
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
||||||
|
else:
|
||||||
|
ok = ok and bool(result.get("ok"))
|
||||||
|
results.append(result)
|
||||||
|
finally:
|
||||||
|
if not args.keep_runtime and args.runtime_root is None:
|
||||||
|
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||||
|
|
||||||
|
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
||||||
|
if args.format == "json":
|
||||||
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||||
|
else:
|
||||||
|
for result in results:
|
||||||
|
status = "ok" if result.get("ok") else "FAILED"
|
||||||
|
print(f"{status}: {result['scenario']}")
|
||||||
|
if result.get("run_id"):
|
||||||
|
print(f" run: {result['run_id']}")
|
||||||
|
if result.get("record_path"):
|
||||||
|
print(f" record: {result['record_path']}")
|
||||||
|
if result.get("error"):
|
||||||
|
print(f" error: {result['error']}")
|
||||||
|
print(f"runtime: {runtime_root}")
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
def package_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "package-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_package_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
)
|
||||||
|
restored_plan = saved_module_install_plan(session)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"package-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def migration_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "migration-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
migrate_database=True,
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"migration-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def restart_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "restart-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
restart_command="restart-fails",
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"restart-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def health_timeout(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "health-timeout-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
restart_command="restart-ok",
|
||||||
|
health_url="http://127.0.0.1:9/health",
|
||||||
|
health_timeout_seconds=0.2,
|
||||||
|
health_interval_seconds=0.05,
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"health-timeout",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
available = {"retirement-example": _retirement_failure_manifest()}
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": "retirement-example",
|
||||||
|
"action": "uninstall",
|
||||||
|
"python_package": "govoplan-retirement-example",
|
||||||
|
"destroy_data": True,
|
||||||
|
}])
|
||||||
|
session.commit()
|
||||||
|
with _patch_subprocess(_success_run):
|
||||||
|
result = run_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available,
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"destructive-retirement-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
||||||
|
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_hooks(root: Path) -> dict[str, object]:
|
||||||
|
_, database = _sqlite_database(root)
|
||||||
|
backup_script = (
|
||||||
|
"import json, os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
check_script = (
|
||||||
|
"import os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
restore_script = (
|
||||||
|
"import os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
||||||
|
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": "postgres-hook-example",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-postgres-hook-example",
|
||||||
|
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
||||||
|
}])
|
||||||
|
session.commit()
|
||||||
|
result = run_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url="postgresql://db.example.invalid/govoplan",
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
migrate_database=True,
|
||||||
|
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
||||||
|
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
||||||
|
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
||||||
|
dry_run=True,
|
||||||
|
)
|
||||||
|
with _patch_subprocess(_non_shell_success_run):
|
||||||
|
rollback = rollback_module_install_run(
|
||||||
|
run_id=result.run_id,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
database_url="postgresql://db.example.invalid/govoplan",
|
||||||
|
)
|
||||||
|
run_dir = result.record_path.parent
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
ok = (
|
||||||
|
result.status == "dry-run"
|
||||||
|
and rollback.status == "rolled-back"
|
||||||
|
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
||||||
|
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
||||||
|
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"scenario": "postgres-hooks",
|
||||||
|
"ok": ok,
|
||||||
|
"root": str(root),
|
||||||
|
"run_id": result.run_id,
|
||||||
|
"record_path": str(result.record_path),
|
||||||
|
"rollback_status": rollback.status,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def queue_and_lock(root: Path) -> dict[str, object]:
|
||||||
|
runtime_dir = root / "installer"
|
||||||
|
status = update_module_installer_daemon_status(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
patch={"status": "polling", "current_request_id": None},
|
||||||
|
)
|
||||||
|
request = queue_module_installer_request(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
requested_by="drill",
|
||||||
|
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
||||||
|
)
|
||||||
|
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
||||||
|
assert claimed is not None
|
||||||
|
failed = update_module_installer_request(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
request_id=str(claimed["request_id"]),
|
||||||
|
patch={"status": "failed", "error": "simulated drill failure"},
|
||||||
|
)
|
||||||
|
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
||||||
|
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
||||||
|
lock_path = runtime_dir / "install.lock"
|
||||||
|
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
||||||
|
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
||||||
|
lock_path.unlink()
|
||||||
|
return {
|
||||||
|
"scenario": "queue-and-lock",
|
||||||
|
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
||||||
|
"root": str(root),
|
||||||
|
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
||||||
|
database_url = f"sqlite:///{root / 'drill.db'}"
|
||||||
|
configure_database(database_url)
|
||||||
|
database = get_database()
|
||||||
|
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
||||||
|
return database_url, database
|
||||||
|
|
||||||
|
|
||||||
|
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": module_id,
|
||||||
|
"action": "install",
|
||||||
|
"python_package": f"govoplan-{module_id}",
|
||||||
|
"python_ref": f"govoplan-{module_id}==0.1.0",
|
||||||
|
}])
|
||||||
|
save_desired_enabled_modules(session, ("tenancy", "access"))
|
||||||
|
session.commit()
|
||||||
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
def _retirement_failure_manifest() -> ModuleManifest:
|
||||||
|
def destroy(_session: object, _module_id: str) -> None:
|
||||||
|
raise RuntimeError("simulated destructive retirement failure")
|
||||||
|
|
||||||
|
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
||||||
|
return MigrationRetirementPlan(
|
||||||
|
supported=True,
|
||||||
|
summary="Drill retirement provider supports destructive cleanup.",
|
||||||
|
destroy_data_supported=True,
|
||||||
|
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
||||||
|
destroy_data_executor=destroy,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ModuleManifest(
|
||||||
|
id="retirement-example",
|
||||||
|
name="Retirement example",
|
||||||
|
version="0.1.0",
|
||||||
|
migration_spec=MigrationSpec(
|
||||||
|
module_id="retirement-example",
|
||||||
|
retirement_supported=True,
|
||||||
|
retirement_provider=provider,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _scenario_result(
|
||||||
|
scenario: str,
|
||||||
|
root: Path,
|
||||||
|
result: Any,
|
||||||
|
*,
|
||||||
|
expected_status: str,
|
||||||
|
checks: dict[str, bool],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"scenario": scenario,
|
||||||
|
"ok": result.status == expected_status and all(checks.values()),
|
||||||
|
"root": str(root),
|
||||||
|
"run_id": result.run_id,
|
||||||
|
"record_path": str(result.record_path),
|
||||||
|
"status": result.status,
|
||||||
|
"checks": checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
||||||
|
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _nested(value: dict[str, object], *keys: str) -> object:
|
||||||
|
current: object = value
|
||||||
|
for key in keys:
|
||||||
|
if not isinstance(current, dict):
|
||||||
|
return None
|
||||||
|
current = current.get(key)
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
|
def _command_text(argv: object) -> str:
|
||||||
|
if isinstance(argv, list | tuple):
|
||||||
|
return " ".join(str(item) for item in argv)
|
||||||
|
return str(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "pip freeze" in text:
|
||||||
|
return _completed(stdout="govoplan-core==0.0.0\n")
|
||||||
|
return _completed()
|
||||||
|
|
||||||
|
|
||||||
|
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
||||||
|
if kwargs.get("shell"):
|
||||||
|
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "pip install" in text and "==" in text and "-r" not in text:
|
||||||
|
return _completed(23, stderr="simulated package install failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "govoplan_core.commands.init_db" in text:
|
||||||
|
return _completed(24, stderr="simulated migration failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
||||||
|
return _completed(25, stderr="simulated restart failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
||||||
|
import govoplan_core.core.module_installer as installer
|
||||||
|
|
||||||
|
original = installer.subprocess.run
|
||||||
|
installer.subprocess.run = fake_run # type: ignore[assignment]
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
installer.subprocess.run = original # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _patch_urlopen() -> Iterable[None]:
|
||||||
|
import govoplan_core.core.module_installer as installer
|
||||||
|
|
||||||
|
original = installer.urllib.request.urlopen
|
||||||
|
|
||||||
|
def fail(*_args: object, **_kwargs: object) -> None:
|
||||||
|
raise URLError("simulated health timeout")
|
||||||
|
|
||||||
|
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
SCENARIOS: dict[str, Scenario] = {
|
||||||
|
"destructive-retirement-failure": destructive_retirement_failure,
|
||||||
|
"health-timeout": health_timeout,
|
||||||
|
"migration-failure": migration_failure,
|
||||||
|
"package-failure": package_failure,
|
||||||
|
"postgres-hooks": postgres_hooks,
|
||||||
|
"queue-and-lock": queue_and_lock,
|
||||||
|
"restart-failure": restart_failure,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
128
scripts/postgres-integration-check.py
Normal file
128
scripts/postgres-integration-check.py
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SRC = ROOT / "src"
|
||||||
|
|
||||||
|
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
||||||
|
("core-only", "tenancy,access,admin"),
|
||||||
|
("files-only", "tenancy,access,admin,files"),
|
||||||
|
("mail-only", "tenancy,access,admin,mail"),
|
||||||
|
("campaign-only", "tenancy,access,admin,campaigns"),
|
||||||
|
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
||||||
|
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
||||||
|
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--database-url",
|
||||||
|
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
||||||
|
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--module-set",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
metavar="NAME=MODULES",
|
||||||
|
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--reset-schema",
|
||||||
|
action="store_true",
|
||||||
|
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
||||||
|
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
||||||
|
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
||||||
|
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
database_url = (args.database_url or "").strip()
|
||||||
|
if not database_url:
|
||||||
|
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
||||||
|
if not database_url.startswith("postgresql"):
|
||||||
|
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
||||||
|
|
||||||
|
module_sets = _parse_module_sets(args.module_set)
|
||||||
|
ok = True
|
||||||
|
for name, modules in module_sets:
|
||||||
|
print(f"\n== {name}: {modules} ==")
|
||||||
|
if args.reset_schema:
|
||||||
|
_reset_public_schema(database_url)
|
||||||
|
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
||||||
|
if not args.skip_migrate:
|
||||||
|
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
||||||
|
if not args.skip_smoke:
|
||||||
|
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
||||||
|
if not raw:
|
||||||
|
return DEFAULT_MODULE_SETS
|
||||||
|
parsed: list[tuple[str, str]] = []
|
||||||
|
for item in raw:
|
||||||
|
name, separator, modules = item.partition("=")
|
||||||
|
if not separator or not name.strip() or not modules.strip():
|
||||||
|
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
||||||
|
parsed.append((name.strip(), modules.strip()))
|
||||||
|
return tuple(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["APP_ENV"] = app_env
|
||||||
|
env["DATABASE_URL"] = database_url
|
||||||
|
env["ENABLED_MODULES"] = modules
|
||||||
|
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
||||||
|
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
||||||
|
env["CELERY_ENABLED"] = "false"
|
||||||
|
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
||||||
|
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
||||||
|
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
||||||
|
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
||||||
|
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
||||||
|
print("+ " + " ".join(command))
|
||||||
|
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_public_schema(database_url: str) -> None:
|
||||||
|
try:
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
except ModuleNotFoundError as exc:
|
||||||
|
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
||||||
|
|
||||||
|
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
||||||
|
connection.exec_driver_sql("CREATE SCHEMA public")
|
||||||
|
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
print("Reset PostgreSQL schema: public")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
113
scripts/production-like-dev.sh
Normal file
113
scripts/production-like-dev.sh
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||||
|
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||||
|
if [ ! -f "$PROFILE_ENV" ]; then
|
||||||
|
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||||
|
fi
|
||||||
|
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
DOCKER="${DOCKER:-docker}"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: scripts/production-like-dev.sh <command> [--yes]
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
start Start the production-like API, worker, WebUI, PostgreSQL, and Redis profile.
|
||||||
|
stop Stop PostgreSQL and Redis profile containers.
|
||||||
|
status Show profile container status and validate the effective environment.
|
||||||
|
seed Run migrations and seed development data against the profile database.
|
||||||
|
reset --yes Stop containers, remove profile volumes, and remove runtime/production-like data.
|
||||||
|
validate-config Validate the profile environment only.
|
||||||
|
|
||||||
|
The start command delegates to scripts/launch-production-like-dev.sh. Stop/reset
|
||||||
|
operate on Docker dependencies; API/WebUI/worker processes started by the
|
||||||
|
launcher are stopped with Ctrl+C in that launcher terminal.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
printf 'production-like-dev: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
command="${1:-}"
|
||||||
|
if [ -z "$command" ] || [ "$command" = "-h" ] || [ "$command" = "--help" ]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
shift || true
|
||||||
|
|
||||||
|
yes=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--yes) yes=1 ;;
|
||||||
|
*) fail "unknown argument: $arg" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||||
|
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$PROFILE_ENV"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
export APP_ENV="${APP_ENV:-staging}"
|
||||||
|
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||||
|
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||||
|
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||||
|
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||||
|
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||||
|
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||||
|
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||||
|
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}"
|
||||||
|
|
||||||
|
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||||
|
MASTER_KEY_B64="$("$PYTHON" -m govoplan_core.commands.config env-template --profile production-like --generate-secrets | sed -n 's/^MASTER_KEY_B64=//p' | head -n 1)"
|
||||||
|
export MASTER_KEY_B64
|
||||||
|
fi
|
||||||
|
|
||||||
|
compose() {
|
||||||
|
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_config() {
|
||||||
|
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$command" in
|
||||||
|
start)
|
||||||
|
exec "$ROOT/scripts/launch-production-like-dev.sh"
|
||||||
|
;;
|
||||||
|
stop)
|
||||||
|
compose down
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
compose ps
|
||||||
|
validate_config
|
||||||
|
;;
|
||||||
|
seed)
|
||||||
|
validate_config
|
||||||
|
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||||
|
;;
|
||||||
|
reset)
|
||||||
|
[ "$yes" = "1" ] || fail "reset is destructive; rerun with --yes"
|
||||||
|
compose down -v
|
||||||
|
rm -rf "$ROOT/runtime/production-like"
|
||||||
|
;;
|
||||||
|
validate-config)
|
||||||
|
validate_config
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage >&2
|
||||||
|
fail "unknown command: $command"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
259
scripts/publish-release-catalog.sh
Normal file
259
scripts/publish-release-catalog.sh
Normal file
@@ -0,0 +1,259 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
|
||||||
|
|
||||||
|
Generates a signed module package catalog for a GovOPlaN release, writes it into
|
||||||
|
govoplan-web/public/catalogs/v1, validates it, optionally builds the website,
|
||||||
|
and optionally commits/tags/pushes govoplan-web.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--version <x.y.z> Release version without leading v.
|
||||||
|
--channel <name> Catalog channel. Defaults to stable.
|
||||||
|
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
||||||
|
--expires-days <days> Catalog expiry window. Defaults to 90.
|
||||||
|
--catalog-signing-key <key-id=/path/private.pem>
|
||||||
|
Ed25519 private key. May be repeated for rotation.
|
||||||
|
--core-root <path> govoplan-core checkout. Defaults to this script parent.
|
||||||
|
--web-root <path> govoplan-web checkout. Defaults to ../govoplan-web.
|
||||||
|
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
|
||||||
|
--npm <path> npm executable used for optional web build.
|
||||||
|
--build-web Run npm run build in govoplan-web after generating files.
|
||||||
|
--commit Commit generated catalog files in govoplan-web.
|
||||||
|
--tag Tag govoplan-web with catalog-v<x.y.z>. Implies --commit.
|
||||||
|
--push Push govoplan-web branch and tag. Implies --commit.
|
||||||
|
--remote <name> Git remote for push. Defaults to origin.
|
||||||
|
--branch <name> Branch to push. Defaults to current govoplan-web branch.
|
||||||
|
-n, --dry-run Print commands and validate inputs without writing git changes.
|
||||||
|
-h, --help Show this help.
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
echo "error: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PARENT="$(dirname "$ROOT")"
|
||||||
|
CORE_ROOT="$ROOT"
|
||||||
|
WEB_ROOT="$PARENT/govoplan-web"
|
||||||
|
VERSION=""
|
||||||
|
CHANNEL="stable"
|
||||||
|
SEQUENCE=""
|
||||||
|
EXPIRES_DAYS="90"
|
||||||
|
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
||||||
|
REMOTE="origin"
|
||||||
|
BRANCH=""
|
||||||
|
BUILD_WEB=0
|
||||||
|
COMMIT=0
|
||||||
|
TAG_WEB=0
|
||||||
|
PUSH=0
|
||||||
|
DRY_RUN=0
|
||||||
|
SIGNING_KEYS=()
|
||||||
|
NPM_BIN="${NPM:-}"
|
||||||
|
|
||||||
|
if [[ -z "${PYTHON:-}" ]]; then
|
||||||
|
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||||
|
PYTHON="$ROOT/.venv/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="python3"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$NPM_BIN" ]]; then
|
||||||
|
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||||
|
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||||
|
else
|
||||||
|
NPM_BIN="npm"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--version)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
VERSION="${2#v}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--channel)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CHANNEL="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--sequence)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
SEQUENCE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--expires-days)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
EXPIRES_DAYS="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-signing-key)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
SIGNING_KEYS+=("$2")
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--core-root)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CORE_ROOT="$(cd "$2" && pwd)"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--web-root)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
WEB_ROOT="$(cd "$2" && pwd)"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--public-base-url)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
PUBLIC_BASE_URL="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--npm)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
NPM_BIN="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--build-web)
|
||||||
|
BUILD_WEB=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--commit)
|
||||||
|
COMMIT=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--tag)
|
||||||
|
TAG_WEB=1
|
||||||
|
COMMIT=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--push)
|
||||||
|
PUSH=1
|
||||||
|
COMMIT=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--remote)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
REMOTE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--branch)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
BRANCH="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-n|--dry-run)
|
||||||
|
DRY_RUN=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown argument: $1" >&2
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[[ -n "$VERSION" ]] || fail "--version is required"
|
||||||
|
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
||||||
|
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
||||||
|
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
||||||
|
[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT"
|
||||||
|
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
||||||
|
|
||||||
|
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||||
|
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$BRANCH" ]]; then
|
||||||
|
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
||||||
|
fi
|
||||||
|
[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch"
|
||||||
|
|
||||||
|
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
||||||
|
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
||||||
|
TAG_NAME="catalog-v$VERSION"
|
||||||
|
|
||||||
|
run() {
|
||||||
|
printf '+'
|
||||||
|
printf ' %q' "$@"
|
||||||
|
printf '\n'
|
||||||
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
|
"$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
GEN_ARGS=(
|
||||||
|
"$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py"
|
||||||
|
--version "$VERSION"
|
||||||
|
--channel "$CHANNEL"
|
||||||
|
--expires-days "$EXPIRES_DAYS"
|
||||||
|
--catalog-output "$CATALOG_PATH"
|
||||||
|
--keyring-output "$KEYRING_PATH"
|
||||||
|
--public-base-url "$PUBLIC_BASE_URL"
|
||||||
|
)
|
||||||
|
if [[ -n "$SEQUENCE" ]]; then
|
||||||
|
GEN_ARGS+=(--sequence "$SEQUENCE")
|
||||||
|
fi
|
||||||
|
for signing_key in "${SIGNING_KEYS[@]}"; do
|
||||||
|
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
||||||
|
done
|
||||||
|
|
||||||
|
run "${GEN_ARGS[@]}"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
|
||||||
|
"$PYTHON" -m govoplan_core.commands.module_installer \
|
||||||
|
--validate-package-catalog "$CATALOG_PATH" \
|
||||||
|
--require-signed-catalog \
|
||||||
|
--approved-catalog-channel "$CHANNEL" \
|
||||||
|
--format json >/dev/null
|
||||||
|
else
|
||||||
|
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||||
|
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$COMMIT" -eq 1 ]]; then
|
||||||
|
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
|
||||||
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
|
if git -C "$WEB_ROOT" diff --cached --quiet; then
|
||||||
|
echo "No govoplan-web catalog changes to commit."
|
||||||
|
else
|
||||||
|
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||||
|
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
|
||||||
|
fail "govoplan-web tag already exists: $TAG_NAME"
|
||||||
|
fi
|
||||||
|
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$PUSH" -eq 1 ]]; then
|
||||||
|
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||||
|
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
|
||||||
|
else
|
||||||
|
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Catalog ready:"
|
||||||
|
echo " $CATALOG_PATH"
|
||||||
|
echo " $KEYRING_PATH"
|
||||||
|
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"
|
||||||
@@ -6,8 +6,9 @@ usage() {
|
|||||||
Usage:
|
Usage:
|
||||||
scripts/push-release-tag.sh [options]
|
scripts/push-release-tag.sh [options]
|
||||||
|
|
||||||
Bumps all GovOPlaN package versions, commits all changes in all four repos,
|
Bumps installable GovOPlaN package versions, commits pending changes in the
|
||||||
creates an annotated release tag in each repo, and pushes branch+tag.
|
GovOPlaN release/module repositories, creates an annotated release tag in each
|
||||||
|
repo, and pushes branch+tag.
|
||||||
|
|
||||||
By default the script asks which version part to bump:
|
By default the script asks which version part to bump:
|
||||||
major X.Y.Z -> (X+1).0.0
|
major X.Y.Z -> (X+1).0.0
|
||||||
@@ -16,20 +17,63 @@ By default the script asks which version part to bump:
|
|||||||
|
|
||||||
Options:
|
Options:
|
||||||
--bump <kind> Version bump: major, minor, subversion, or patch.
|
--bump <kind> Version bump: major, minor, subversion, or patch.
|
||||||
--version <x.y.z> Explicit target version instead of a bump.
|
--version <x.y.z> Explicit target version instead of a bump. May equal
|
||||||
|
the current repo versions when they were bumped already.
|
||||||
-r, --remote <name> Remote to push to. Defaults to origin.
|
-r, --remote <name> Remote to push to. Defaults to origin.
|
||||||
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
|
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
|
||||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||||
|
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||||
|
--warn-migration-audit Run the migration release audit without baseline strictness.
|
||||||
|
--strict-migration-audit
|
||||||
|
Require current Alembic heads to be recorded in the
|
||||||
|
latest migration release baseline.
|
||||||
|
--skip-migration-audit Do not run the release migration audit.
|
||||||
|
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||||
|
catalog into govoplan-web.
|
||||||
|
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||||
|
Defaults to ../govoplan-web.
|
||||||
|
--catalog-signing-key <key-id=/path/private.pem>
|
||||||
|
Catalog signing key for --publish-web-catalog. May be
|
||||||
|
repeated during key rotation.
|
||||||
|
--catalog-channel <name>
|
||||||
|
Catalog channel. Defaults to stable.
|
||||||
|
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
|
||||||
|
--catalog-expires-days <days>
|
||||||
|
Catalog expiry window. Defaults to 90.
|
||||||
|
--catalog-public-url <url>
|
||||||
|
Public website URL. Defaults to
|
||||||
|
https://govoplan.add-ideas.de.
|
||||||
|
--build-web-catalog Run npm run build in govoplan-web before committing the
|
||||||
|
catalog update.
|
||||||
|
--npm <path> npm executable for lockfile generation.
|
||||||
-y, --yes Do not prompt before committing, tagging, and pushing.
|
-y, --yes Do not prompt before committing, tagging, and pushing.
|
||||||
-n, --dry-run Print intended actions without changing files or git state.
|
-n, --dry-run Print intended actions without changing files or git state.
|
||||||
-h, --help Show this help.
|
-h, --help Show this help.
|
||||||
|
|
||||||
Repos:
|
Repos:
|
||||||
govoplan-core
|
Installable release packages:
|
||||||
|
govoplan-access
|
||||||
|
govoplan-admin
|
||||||
|
govoplan-tenancy
|
||||||
|
govoplan-organizations
|
||||||
|
govoplan-identity
|
||||||
|
govoplan-idm
|
||||||
|
govoplan-policy
|
||||||
|
govoplan-audit
|
||||||
|
govoplan-dashboard
|
||||||
govoplan-files
|
govoplan-files
|
||||||
govoplan-mail
|
govoplan-mail
|
||||||
govoplan-campaign
|
govoplan-campaign
|
||||||
|
govoplan-calendar
|
||||||
|
govoplan-ops
|
||||||
|
govoplan-core
|
||||||
|
|
||||||
|
Roadmap/scaffold module repositories are tag-only until they have package
|
||||||
|
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||||
|
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
||||||
|
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||||
|
xrechnung, and xta-osci.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
scripts/push-release-tag.sh
|
scripts/push-release-tag.sh
|
||||||
@@ -45,10 +89,72 @@ fail() {
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
PARENT="$(dirname "$ROOT")"
|
PARENT="$(dirname "$ROOT")"
|
||||||
REPOS=(
|
PACKAGE_MODULE_REPOS=(
|
||||||
"$PARENT/govoplan-files"
|
"$PARENT/govoplan-access"
|
||||||
"$PARENT/govoplan-mail"
|
"$PARENT/govoplan-admin"
|
||||||
|
"$PARENT/govoplan-approvals"
|
||||||
|
"$PARENT/govoplan-assets"
|
||||||
|
"$PARENT/govoplan-audit"
|
||||||
|
"$PARENT/govoplan-booking"
|
||||||
|
"$PARENT/govoplan-calendar"
|
||||||
"$PARENT/govoplan-campaign"
|
"$PARENT/govoplan-campaign"
|
||||||
|
"$PARENT/govoplan-certificates"
|
||||||
|
"$PARENT/govoplan-committee"
|
||||||
|
"$PARENT/govoplan-consultation"
|
||||||
|
"$PARENT/govoplan-contracts"
|
||||||
|
"$PARENT/govoplan-dashboard"
|
||||||
|
"$PARENT/govoplan-docs"
|
||||||
|
"$PARENT/govoplan-facilities"
|
||||||
|
"$PARENT/govoplan-files"
|
||||||
|
"$PARENT/govoplan-forms-runtime"
|
||||||
|
"$PARENT/govoplan-grants"
|
||||||
|
"$PARENT/govoplan-helpdesk"
|
||||||
|
"$PARENT/govoplan-identity"
|
||||||
|
"$PARENT/govoplan-idm"
|
||||||
|
"$PARENT/govoplan-inspections"
|
||||||
|
"$PARENT/govoplan-issue-reporting"
|
||||||
|
"$PARENT/govoplan-learning"
|
||||||
|
"$PARENT/govoplan-mail"
|
||||||
|
"$PARENT/govoplan-ops"
|
||||||
|
"$PARENT/govoplan-organizations"
|
||||||
|
"$PARENT/govoplan-permits"
|
||||||
|
"$PARENT/govoplan-policy"
|
||||||
|
"$PARENT/govoplan-procurement"
|
||||||
|
"$PARENT/govoplan-records"
|
||||||
|
"$PARENT/govoplan-resources"
|
||||||
|
"$PARENT/govoplan-risk-compliance"
|
||||||
|
"$PARENT/govoplan-tenancy"
|
||||||
|
"$PARENT/govoplan-transparency"
|
||||||
|
)
|
||||||
|
TAG_ONLY_MODULE_REPOS=(
|
||||||
|
"$PARENT/govoplan-addresses"
|
||||||
|
"$PARENT/govoplan-appointments"
|
||||||
|
"$PARENT/govoplan-cases"
|
||||||
|
"$PARENT/govoplan-connectors"
|
||||||
|
"$PARENT/govoplan-dms"
|
||||||
|
"$PARENT/govoplan-erp"
|
||||||
|
"$PARENT/govoplan-fit-connect"
|
||||||
|
"$PARENT/govoplan-forms"
|
||||||
|
"$PARENT/govoplan-identity-trust"
|
||||||
|
"$PARENT/govoplan-ledger"
|
||||||
|
"$PARENT/govoplan-notifications"
|
||||||
|
"$PARENT/govoplan-payments"
|
||||||
|
"$PARENT/govoplan-portal"
|
||||||
|
"$PARENT/govoplan-postbox"
|
||||||
|
"$PARENT/govoplan-reporting"
|
||||||
|
"$PARENT/govoplan-scheduling"
|
||||||
|
"$PARENT/govoplan-search"
|
||||||
|
"$PARENT/govoplan-tasks"
|
||||||
|
"$PARENT/govoplan-templates"
|
||||||
|
"$PARENT/govoplan-workflow"
|
||||||
|
"$PARENT/govoplan-xoev"
|
||||||
|
"$PARENT/govoplan-xrechnung"
|
||||||
|
"$PARENT/govoplan-xta-osci"
|
||||||
|
)
|
||||||
|
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||||
|
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
||||||
|
REPOS=(
|
||||||
|
"${MODULE_REPOS[@]}"
|
||||||
"$ROOT"
|
"$ROOT"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -60,6 +166,17 @@ COMMIT_MESSAGE=""
|
|||||||
TAG_MESSAGE=""
|
TAG_MESSAGE=""
|
||||||
DRY_RUN=0
|
DRY_RUN=0
|
||||||
YES=0
|
YES=0
|
||||||
|
GENERATE_RELEASE_LOCK=1
|
||||||
|
MIGRATION_AUDIT_MODE="auto"
|
||||||
|
NPM_BIN="${NPM:-}"
|
||||||
|
PUBLISH_WEB_CATALOG=0
|
||||||
|
WEB_ROOT="$PARENT/govoplan-web"
|
||||||
|
CATALOG_CHANNEL="stable"
|
||||||
|
CATALOG_SEQUENCE=""
|
||||||
|
CATALOG_EXPIRES_DAYS="90"
|
||||||
|
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
|
||||||
|
BUILD_WEB_CATALOG=0
|
||||||
|
CATALOG_SIGNING_KEYS=()
|
||||||
|
|
||||||
if [[ -z "${PYTHON:-}" ]]; then
|
if [[ -z "${PYTHON:-}" ]]; then
|
||||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||||
@@ -69,6 +186,17 @@ if [[ -z "${PYTHON:-}" ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$NPM_BIN" ]]; then
|
||||||
|
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||||
|
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||||
|
else
|
||||||
|
NPM_BIN="npm"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o BatchMode=yes -o ConnectTimeout=15}"
|
||||||
|
GIT_REMOTE_TIMEOUT="${GIT_REMOTE_TIMEOUT:-30}"
|
||||||
|
|
||||||
normalize_bump() {
|
normalize_bump() {
|
||||||
local value="${1,,}"
|
local value="${1,,}"
|
||||||
case "$value" in
|
case "$value" in
|
||||||
@@ -162,8 +290,8 @@ if version_count != 1:
|
|||||||
|
|
||||||
if name != "govoplan-core":
|
if name != "govoplan-core":
|
||||||
text, dependency_count = re.subn(
|
text, dependency_count = re.subn(
|
||||||
r'govoplan-core>=\d+\.\d+\.\d+',
|
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
|
||||||
f'govoplan-core>={new_version}',
|
rf'\g<1>{new_version}',
|
||||||
text,
|
text,
|
||||||
)
|
)
|
||||||
if dependency_count < 1:
|
if dependency_count < 1:
|
||||||
@@ -174,6 +302,158 @@ print(f"{name}: {old_version} -> {new_version}")
|
|||||||
PYCODE
|
PYCODE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
update_manifest_version() {
|
||||||
|
local repo="$1"
|
||||||
|
local project_name="$2"
|
||||||
|
local version="$3"
|
||||||
|
local manifest_paths=()
|
||||||
|
|
||||||
|
if [[ "$project_name" == "govoplan-core" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS= read -r -d '' manifest_path; do
|
||||||
|
manifest_paths+=("$manifest_path")
|
||||||
|
done < <(find "$repo/src" -path "*/backend/manifest.py" -print0 2>/dev/null)
|
||||||
|
|
||||||
|
[[ "${#manifest_paths[@]}" -gt 0 ]] || fail "missing module manifest under $repo/src"
|
||||||
|
|
||||||
|
for manifest_path in "${manifest_paths[@]}"; do
|
||||||
|
"$PYTHON" - "$manifest_path" "$version" <<'PYCODE'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path = pathlib.Path(sys.argv[1])
|
||||||
|
new_version = sys.argv[2]
|
||||||
|
text = path.read_text()
|
||||||
|
text, count = re.subn(
|
||||||
|
r'(?m)^(\s*version=)["\'][^"\']+["\'](,?\s*)$',
|
||||||
|
rf'\1"{new_version}"\2',
|
||||||
|
text,
|
||||||
|
count=1,
|
||||||
|
)
|
||||||
|
if count == 0:
|
||||||
|
text, count = re.subn(
|
||||||
|
r'(?m)^(MODULE_VERSION\s*=\s*)["\'][^"\']+["\'](\s*)$',
|
||||||
|
rf'\1"{new_version}"\2',
|
||||||
|
text,
|
||||||
|
count=1,
|
||||||
|
)
|
||||||
|
if count != 1:
|
||||||
|
raise SystemExit(f"could not update ModuleManifest.version in {path}")
|
||||||
|
path.write_text(text)
|
||||||
|
PYCODE
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
update_webui_package() {
|
||||||
|
local repo="$1"
|
||||||
|
local project_name="$2"
|
||||||
|
local version="$3"
|
||||||
|
local package_path=""
|
||||||
|
local release_package_path="$repo/webui/package.release.json"
|
||||||
|
|
||||||
|
for package_path in "$repo/package.json" "$repo/webui/package.json"; do
|
||||||
|
[[ -f "$package_path" ]] || continue
|
||||||
|
"$PYTHON" - "$package_path" "$project_name" "$version" <<'PYCODE'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path = pathlib.Path(sys.argv[1])
|
||||||
|
project_name = sys.argv[2]
|
||||||
|
new_version = sys.argv[3]
|
||||||
|
|
||||||
|
data = json.loads(path.read_text())
|
||||||
|
data["version"] = new_version
|
||||||
|
if project_name != "govoplan-core":
|
||||||
|
peers = data.setdefault("peerDependencies", {})
|
||||||
|
if "@govoplan/core-webui" in peers:
|
||||||
|
peers["@govoplan/core-webui"] = f"^{new_version}"
|
||||||
|
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||||
|
PYCODE
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
||||||
|
"$PYTHON" - "$release_package_path" "$version" <<'PYCODE'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
path = pathlib.Path(sys.argv[1])
|
||||||
|
new_version = sys.argv[2]
|
||||||
|
|
||||||
|
data = json.loads(path.read_text())
|
||||||
|
data["version"] = new_version
|
||||||
|
dependencies = data.setdefault("dependencies", {})
|
||||||
|
for name, spec in list(dependencies.items()):
|
||||||
|
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
|
||||||
|
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||||
|
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||||
|
PYCODE
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
update_version_files() {
|
||||||
|
local repo="$1"
|
||||||
|
local version="$2"
|
||||||
|
local project_name="${PROJECT_NAMES[$repo]}"
|
||||||
|
|
||||||
|
update_pyproject "$repo" "$version"
|
||||||
|
update_manifest_version "$repo" "$project_name" "$version"
|
||||||
|
update_webui_package "$repo" "$project_name" "$version"
|
||||||
|
}
|
||||||
|
|
||||||
|
refresh_development_webui_lock() {
|
||||||
|
[[ -f "$ROOT/webui/package.json" ]] || return 0
|
||||||
|
[[ -f "$ROOT/webui/package-lock.json" ]] || return 0
|
||||||
|
|
||||||
|
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$ROOT/webui" install --package-lock-only --ignore-scripts
|
||||||
|
}
|
||||||
|
|
||||||
|
generate_release_lock() {
|
||||||
|
if [[ "$GENERATE_RELEASE_LOCK" -eq 0 ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ -x "$ROOT/scripts/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $ROOT/scripts/generate-release-lock.sh"
|
||||||
|
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_migration_release_audit() {
|
||||||
|
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
||||||
|
local command=("$PYTHON" "$audit_script")
|
||||||
|
|
||||||
|
case "$MIGRATION_AUDIT_MODE" in
|
||||||
|
skip)
|
||||||
|
echo "Migration release audit: skipped"
|
||||||
|
return
|
||||||
|
;;
|
||||||
|
strict)
|
||||||
|
command+=("--strict")
|
||||||
|
;;
|
||||||
|
auto)
|
||||||
|
command+=("--strict-if-baseline")
|
||||||
|
;;
|
||||||
|
warn)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
||||||
|
run "${command[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
print_command() {
|
print_command() {
|
||||||
printf '+'
|
printf '+'
|
||||||
printf ' %q' "$@"
|
printf ' %q' "$@"
|
||||||
@@ -276,6 +556,65 @@ while [[ $# -gt 0 ]]; do
|
|||||||
TAG_MESSAGE="$2"
|
TAG_MESSAGE="$2"
|
||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
|
--skip-release-lock)
|
||||||
|
GENERATE_RELEASE_LOCK=0
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--warn-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="warn"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--strict-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="strict"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--skip-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="skip"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--publish-web-catalog)
|
||||||
|
PUBLISH_WEB_CATALOG=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--web-root)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
WEB_ROOT="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-signing-key)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CATALOG_SIGNING_KEYS+=("$2")
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-channel)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CATALOG_CHANNEL="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-sequence)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CATALOG_SEQUENCE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-expires-days)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CATALOG_EXPIRES_DAYS="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--catalog-public-url)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
CATALOG_PUBLIC_URL="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--build-web-catalog)
|
||||||
|
BUILD_WEB_CATALOG=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--npm)
|
||||||
|
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||||
|
NPM_BIN="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
-y|--yes)
|
-y|--yes)
|
||||||
YES=1
|
YES=1
|
||||||
shift
|
shift
|
||||||
@@ -299,25 +638,42 @@ done
|
|||||||
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
||||||
fail "use either --bump or --version, not both"
|
fail "use either --bump or --version, not both"
|
||||||
fi
|
fi
|
||||||
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
|
||||||
|
fail "--publish-web-catalog requires at least one --catalog-signing-key"
|
||||||
|
fi
|
||||||
|
|
||||||
prompt_for_bump
|
prompt_for_bump
|
||||||
|
|
||||||
if ! command -v "$PYTHON" >/dev/null 2>&1; then
|
if ! command -v "$PYTHON" >/dev/null 2>&1; then
|
||||||
fail "Python interpreter not found: $PYTHON"
|
fail "Python interpreter not found: $PYTHON"
|
||||||
fi
|
fi
|
||||||
|
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
|
||||||
|
fail "npm executable not found: $NPM_BIN"
|
||||||
|
fi
|
||||||
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||||
|
[[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT"
|
||||||
|
[[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
declare -A PROJECT_NAMES
|
declare -A PROJECT_NAMES
|
||||||
declare -A OLD_VERSIONS
|
declare -A OLD_VERSIONS
|
||||||
declare -A BRANCHES
|
declare -A BRANCHES
|
||||||
|
declare -A REPO_KINDS
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${REPOS[@]}"; do
|
||||||
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
||||||
[[ -f "$repo/pyproject.toml" ]] || fail "missing pyproject.toml: $repo"
|
|
||||||
|
|
||||||
|
if [[ -f "$repo/pyproject.toml" ]]; then
|
||||||
|
REPO_KINDS["$repo"]="package"
|
||||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
||||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
||||||
|
|
||||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
||||||
|
else
|
||||||
|
REPO_KINDS["$repo"]="tag-only"
|
||||||
|
PROJECT_NAMES["$repo"]="$(basename "$repo")"
|
||||||
|
OLD_VERSIONS["$repo"]="tag-only"
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -n "$BRANCH" ]]; then
|
if [[ -n "$BRANCH" ]]; then
|
||||||
BRANCHES["$repo"]="$BRANCH"
|
BRANCHES["$repo"]="$BRANCH"
|
||||||
@@ -330,7 +686,7 @@ for repo in "${REPOS[@]}"; do
|
|||||||
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
||||||
|
|
||||||
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
||||||
if [[ -n "$upstream" ]]; then
|
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
|
||||||
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
||||||
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
||||||
fi
|
fi
|
||||||
@@ -338,7 +694,7 @@ done
|
|||||||
|
|
||||||
if [[ -z "$TARGET_VERSION" ]]; then
|
if [[ -z "$TARGET_VERSION" ]]; then
|
||||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
||||||
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
||||||
fi
|
fi
|
||||||
@@ -348,7 +704,10 @@ fi
|
|||||||
|
|
||||||
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
|
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
|
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
|
||||||
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
|
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
|
||||||
fi
|
fi
|
||||||
@@ -366,7 +725,7 @@ for repo in "${REPOS[@]}"; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
set +e
|
set +e
|
||||||
git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
|
timeout "$GIT_REMOTE_TIMEOUT" git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
|
||||||
ls_remote_status=$?
|
ls_remote_status=$?
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
@@ -382,41 +741,120 @@ echo " version: $TARGET_VERSION"
|
|||||||
echo " tag: $TAG"
|
echo " tag: $TAG"
|
||||||
echo " remote: $REMOTE"
|
echo " remote: $REMOTE"
|
||||||
echo " commit message: $COMMIT_MESSAGE"
|
echo " commit message: $COMMIT_MESSAGE"
|
||||||
|
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||||
|
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
||||||
|
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||||
|
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
||||||
|
else
|
||||||
|
echo " release lock: skipped"
|
||||||
|
fi
|
||||||
|
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
||||||
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||||
|
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||||
|
else
|
||||||
|
echo " web catalog: skipped"
|
||||||
|
fi
|
||||||
echo
|
echo
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${REPOS[@]}"; do
|
||||||
|
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
||||||
|
else
|
||||||
|
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
|
||||||
|
fi
|
||||||
git -C "$repo" status --short
|
git -C "$repo" status --short
|
||||||
echo
|
echo
|
||||||
done
|
done
|
||||||
|
|
||||||
|
run_migration_release_audit
|
||||||
|
|
||||||
confirm_release
|
confirm_release
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
echo "Would update $repo/pyproject.toml to $TARGET_VERSION"
|
echo "Would update version files in $repo to $TARGET_VERSION"
|
||||||
else
|
else
|
||||||
update_pyproject "$repo" "$TARGET_VERSION"
|
update_version_files "$repo" "$TARGET_VERSION"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
refresh_development_webui_lock
|
||||||
|
|
||||||
|
for repo in "${MODULE_REPOS[@]}"; do
|
||||||
run git -C "$repo" add -A
|
run git -C "$repo" add -A
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
if git -C "$repo" diff --cached --quiet; then
|
if git -C "$repo" diff --cached --quiet; then
|
||||||
|
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||||
|
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||||
|
echo "No staged changes for ${PROJECT_NAMES[$repo]}; version is already $TARGET_VERSION, tagging current HEAD."
|
||||||
|
else
|
||||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
||||||
fi
|
fi
|
||||||
|
else
|
||||||
|
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
|
||||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||||
done
|
done
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
for repo in "${MODULE_REPOS[@]}"; do
|
||||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
generate_release_lock
|
||||||
|
|
||||||
|
run git -C "$ROOT" add -A
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||||
|
if git -C "$ROOT" diff --cached --quiet; then
|
||||||
|
fail "no staged changes for ${PROJECT_NAMES[$ROOT]} after version bump"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||||
|
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||||
|
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||||
|
|
||||||
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||||
|
CATALOG_ARGS=(
|
||||||
|
"$ROOT/scripts/publish-release-catalog.sh"
|
||||||
|
--version "$TARGET_VERSION"
|
||||||
|
--channel "$CATALOG_CHANNEL"
|
||||||
|
--expires-days "$CATALOG_EXPIRES_DAYS"
|
||||||
|
--core-root "$ROOT"
|
||||||
|
--web-root "$WEB_ROOT"
|
||||||
|
--public-base-url "$CATALOG_PUBLIC_URL"
|
||||||
|
--npm "$NPM_BIN"
|
||||||
|
--commit
|
||||||
|
--tag
|
||||||
|
--push
|
||||||
|
--remote "$REMOTE"
|
||||||
|
)
|
||||||
|
if [[ -n "$BRANCH" ]]; then
|
||||||
|
CATALOG_ARGS+=(--branch "$BRANCH")
|
||||||
|
fi
|
||||||
|
if [[ -n "$CATALOG_SEQUENCE" ]]; then
|
||||||
|
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
|
||||||
|
fi
|
||||||
|
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
|
||||||
|
CATALOG_ARGS+=(--build-web)
|
||||||
|
fi
|
||||||
|
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
|
||||||
|
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
|
||||||
|
done
|
||||||
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
|
CATALOG_ARGS+=(--dry-run)
|
||||||
|
fi
|
||||||
|
run "${CATALOG_ARGS[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
||||||
else
|
else
|
||||||
|
|||||||
422
scripts/release-migration-audit.py
Normal file
422
scripts/release-migration-audit.py
Normal file
@@ -0,0 +1,422 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import ast
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Migration:
|
||||||
|
owner: str
|
||||||
|
path: Path
|
||||||
|
revision: str
|
||||||
|
down_revisions: tuple[str, ...]
|
||||||
|
branch_labels: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--workspace-root",
|
||||||
|
type=Path,
|
||||||
|
default=ROOT.parent,
|
||||||
|
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--baseline-file",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_BASELINE_FILE,
|
||||||
|
help="JSON file recording released migration heads.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict",
|
||||||
|
action="store_true",
|
||||||
|
help="Fail when current heads are not recorded in the latest release baseline.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict-if-baseline",
|
||||||
|
action="store_true",
|
||||||
|
help="Run in strict mode only after at least one release baseline is recorded.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--record-release",
|
||||||
|
metavar="VERSION",
|
||||||
|
help="Record the current reviewed migration heads as a release baseline.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--replace-release",
|
||||||
|
action="store_true",
|
||||||
|
help="Replace an existing release entry when used with --record-release.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--squash-plan",
|
||||||
|
action="store_true",
|
||||||
|
help="Print the reviewed/manual squash checklist for the current graph.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
migrations = discover_migrations(args.workspace_root)
|
||||||
|
baseline = load_baseline_file(args.baseline_file)
|
||||||
|
report = build_report(
|
||||||
|
migrations,
|
||||||
|
baseline=baseline,
|
||||||
|
baseline_file=args.baseline_file,
|
||||||
|
workspace_root=args.workspace_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.record_release:
|
||||||
|
if report["graph_errors"]:
|
||||||
|
print_text_report(report)
|
||||||
|
return 1
|
||||||
|
baseline = record_release_baseline(
|
||||||
|
baseline,
|
||||||
|
report,
|
||||||
|
release=args.record_release,
|
||||||
|
replace=args.replace_release,
|
||||||
|
)
|
||||||
|
write_baseline_file(args.baseline_file, baseline)
|
||||||
|
report = build_report(
|
||||||
|
migrations,
|
||||||
|
baseline=baseline,
|
||||||
|
baseline_file=args.baseline_file,
|
||||||
|
workspace_root=args.workspace_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.squash_plan:
|
||||||
|
print_squash_plan(report)
|
||||||
|
elif args.json:
|
||||||
|
print(json.dumps(report, indent=2, sort_keys=True))
|
||||||
|
else:
|
||||||
|
print_text_report(report)
|
||||||
|
|
||||||
|
has_graph_errors = bool(report["graph_errors"])
|
||||||
|
has_strict_errors = bool(report["strict_errors"])
|
||||||
|
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
||||||
|
if has_graph_errors or (strict_required and has_strict_errors):
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
||||||
|
migrations: list[Migration] = []
|
||||||
|
for versions_dir in _migration_version_dirs(workspace_root):
|
||||||
|
owner = owner_for_versions_dir(versions_dir)
|
||||||
|
for path in sorted(versions_dir.glob("*.py")):
|
||||||
|
if path.name == "__init__.py":
|
||||||
|
continue
|
||||||
|
migration = parse_migration_file(owner, path)
|
||||||
|
if migration is not None:
|
||||||
|
migrations.append(migration)
|
||||||
|
return migrations
|
||||||
|
|
||||||
|
|
||||||
|
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
||||||
|
candidates = [ROOT / "alembic" / "versions"]
|
||||||
|
for repo in sorted(workspace_root.glob("govoplan-*")):
|
||||||
|
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
||||||
|
seen: set[Path] = set()
|
||||||
|
result: list[Path] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
resolved = candidate.resolve()
|
||||||
|
if resolved in seen or not candidate.is_dir():
|
||||||
|
continue
|
||||||
|
seen.add(resolved)
|
||||||
|
result.append(candidate)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def owner_for_versions_dir(versions_dir: Path) -> str:
|
||||||
|
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
||||||
|
return "govoplan-core"
|
||||||
|
for parent in versions_dir.parents:
|
||||||
|
if parent.name.startswith("govoplan-"):
|
||||||
|
return parent.name
|
||||||
|
return versions_dir.parent.name
|
||||||
|
|
||||||
|
|
||||||
|
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
||||||
|
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||||
|
values: dict[str, Any] = {}
|
||||||
|
for statement in tree.body:
|
||||||
|
if isinstance(statement, ast.Assign):
|
||||||
|
for target in statement.targets:
|
||||||
|
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
||||||
|
values[target.id] = ast.literal_eval(statement.value)
|
||||||
|
elif (
|
||||||
|
isinstance(statement, ast.AnnAssign)
|
||||||
|
and isinstance(statement.target, ast.Name)
|
||||||
|
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
||||||
|
and statement.value is not None
|
||||||
|
):
|
||||||
|
values[statement.target.id] = ast.literal_eval(statement.value)
|
||||||
|
revision = values.get("revision")
|
||||||
|
if not isinstance(revision, str):
|
||||||
|
return None
|
||||||
|
return Migration(
|
||||||
|
owner=owner,
|
||||||
|
path=path,
|
||||||
|
revision=revision,
|
||||||
|
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
||||||
|
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(value, str):
|
||||||
|
return (value,)
|
||||||
|
if isinstance(value, (list, tuple, set)):
|
||||||
|
return tuple(str(item) for item in value if item is not None)
|
||||||
|
return (str(value),)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(value, str):
|
||||||
|
return (value,)
|
||||||
|
if isinstance(value, (list, tuple, set)):
|
||||||
|
return tuple(str(item) for item in value)
|
||||||
|
return (str(value),)
|
||||||
|
|
||||||
|
|
||||||
|
def load_baseline_file(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.exists():
|
||||||
|
return {"version": 1, "releases": [], "_missing": True}
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise SystemExit(f"{path} must contain a JSON object")
|
||||||
|
releases = payload.get("releases")
|
||||||
|
if not isinstance(releases, list):
|
||||||
|
raise SystemExit(f"{path} must contain a releases list")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def build_report(
|
||||||
|
migrations: list[Migration],
|
||||||
|
*,
|
||||||
|
baseline: dict[str, Any],
|
||||||
|
baseline_file: Path,
|
||||||
|
workspace_root: Path,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
revisions: dict[str, Migration] = {}
|
||||||
|
graph_errors: list[str] = []
|
||||||
|
for migration in migrations:
|
||||||
|
existing = revisions.get(migration.revision)
|
||||||
|
if existing is not None:
|
||||||
|
graph_errors.append(
|
||||||
|
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
||||||
|
)
|
||||||
|
revisions[migration.revision] = migration
|
||||||
|
|
||||||
|
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
||||||
|
for migration in migrations:
|
||||||
|
for down_revision in migration.down_revisions:
|
||||||
|
if down_revision not in revisions:
|
||||||
|
graph_errors.append(
|
||||||
|
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
||||||
|
current_head_entries = [
|
||||||
|
{
|
||||||
|
"owner": revisions[revision].owner,
|
||||||
|
"revision": revision,
|
||||||
|
}
|
||||||
|
for revision in current_heads
|
||||||
|
]
|
||||||
|
owner_rows = []
|
||||||
|
for owner in sorted({migration.owner for migration in migrations}):
|
||||||
|
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
||||||
|
owner_revisions = {migration.revision for migration in owner_migrations}
|
||||||
|
owner_referenced = {
|
||||||
|
revision
|
||||||
|
for migration in owner_migrations
|
||||||
|
for revision in migration.down_revisions
|
||||||
|
if revision in owner_revisions
|
||||||
|
}
|
||||||
|
owner_heads = sorted(owner_revisions - owner_referenced)
|
||||||
|
owner_rows.append(
|
||||||
|
{
|
||||||
|
"owner": owner,
|
||||||
|
"migrations": len(owner_migrations),
|
||||||
|
"heads": owner_heads,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
releases = baseline.get("releases") or []
|
||||||
|
latest_release = releases[-1] if releases else None
|
||||||
|
latest_heads = _latest_release_heads(latest_release)
|
||||||
|
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
||||||
|
|
||||||
|
strict_errors: list[str] = []
|
||||||
|
if baseline.get("_missing"):
|
||||||
|
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
||||||
|
if not releases:
|
||||||
|
strict_errors.append("no released migration baseline has been recorded yet")
|
||||||
|
if unrecorded_heads:
|
||||||
|
strict_errors.append(
|
||||||
|
"current migration heads are not recorded in the latest release baseline: "
|
||||||
|
+ ", ".join(unrecorded_heads)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"workspace_root": str(workspace_root),
|
||||||
|
"baseline_file": str(baseline_file),
|
||||||
|
"baseline_file_missing": bool(baseline.get("_missing")),
|
||||||
|
"release_count": len(releases),
|
||||||
|
"latest_release": latest_release,
|
||||||
|
"owners": owner_rows,
|
||||||
|
"current_heads": current_heads,
|
||||||
|
"current_head_entries": current_head_entries,
|
||||||
|
"graph_errors": graph_errors,
|
||||||
|
"strict_errors": strict_errors,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_release_heads(latest_release: Any) -> set[str]:
|
||||||
|
if not isinstance(latest_release, dict):
|
||||||
|
return set()
|
||||||
|
heads = latest_release.get("heads") or []
|
||||||
|
graph_heads = latest_release.get("graph_heads") or []
|
||||||
|
result: set[str] = set()
|
||||||
|
for head_list in (heads, graph_heads):
|
||||||
|
if not isinstance(head_list, list):
|
||||||
|
continue
|
||||||
|
for entry in head_list:
|
||||||
|
if isinstance(entry, str):
|
||||||
|
result.add(entry)
|
||||||
|
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
||||||
|
result.add(entry["revision"])
|
||||||
|
owner_heads = latest_release.get("owner_heads") or []
|
||||||
|
if isinstance(owner_heads, list):
|
||||||
|
for entry in owner_heads:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
revisions = entry.get("revisions") or []
|
||||||
|
if isinstance(revisions, str):
|
||||||
|
result.add(revisions)
|
||||||
|
elif isinstance(revisions, list):
|
||||||
|
result.update(revision for revision in revisions if isinstance(revision, str))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def record_release_baseline(
|
||||||
|
baseline: dict[str, Any],
|
||||||
|
report: dict[str, Any],
|
||||||
|
*,
|
||||||
|
release: str,
|
||||||
|
replace: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
payload = {
|
||||||
|
key: value
|
||||||
|
for key, value in baseline.items()
|
||||||
|
if not key.startswith("_")
|
||||||
|
}
|
||||||
|
payload.setdefault("version", 1)
|
||||||
|
releases = list(payload.get("releases") or [])
|
||||||
|
existing_index = next(
|
||||||
|
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if existing_index is not None and not replace:
|
||||||
|
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"release": release,
|
||||||
|
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||||
|
"squash_policy": "reviewed-manual",
|
||||||
|
"heads": report["current_head_entries"],
|
||||||
|
"owner_heads": [
|
||||||
|
{
|
||||||
|
"owner": owner["owner"],
|
||||||
|
"revisions": owner["heads"],
|
||||||
|
}
|
||||||
|
for owner in report["owners"]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
if existing_index is None:
|
||||||
|
releases.append(entry)
|
||||||
|
else:
|
||||||
|
releases[existing_index] = entry
|
||||||
|
payload["releases"] = releases
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def print_text_report(report: dict[str, Any]) -> None:
|
||||||
|
print("Migration release audit")
|
||||||
|
print(f" workspace: {report['workspace_root']}")
|
||||||
|
print(f" baseline file: {report['baseline_file']}")
|
||||||
|
print(f" recorded releases: {report['release_count']}")
|
||||||
|
print()
|
||||||
|
print("Owners:")
|
||||||
|
for owner in report["owners"]:
|
||||||
|
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||||
|
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
||||||
|
print()
|
||||||
|
print("Current heads:")
|
||||||
|
for entry in report["current_head_entries"]:
|
||||||
|
print(f" {entry['owner']}: {entry['revision']}")
|
||||||
|
if not report["current_head_entries"]:
|
||||||
|
print(" -")
|
||||||
|
|
||||||
|
if report["graph_errors"]:
|
||||||
|
print()
|
||||||
|
print("Graph errors:")
|
||||||
|
for error in report["graph_errors"]:
|
||||||
|
print(f" - {error}")
|
||||||
|
|
||||||
|
if report["strict_errors"]:
|
||||||
|
print()
|
||||||
|
print("Release baseline warnings:")
|
||||||
|
for error in report["strict_errors"]:
|
||||||
|
print(f" - {error}")
|
||||||
|
|
||||||
|
|
||||||
|
def print_squash_plan(report: dict[str, Any]) -> None:
|
||||||
|
print("Manual migration squash plan")
|
||||||
|
print()
|
||||||
|
print("Policy:")
|
||||||
|
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
||||||
|
print(" - Squash only unreleased development migrations.")
|
||||||
|
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
||||||
|
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
||||||
|
print()
|
||||||
|
print("Current owner heads:")
|
||||||
|
for owner in report["owners"]:
|
||||||
|
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||||
|
print(f" - {owner['owner']}: {heads}")
|
||||||
|
print()
|
||||||
|
print("Current Alembic graph heads:")
|
||||||
|
for entry in report["current_head_entries"]:
|
||||||
|
print(f" - {entry['owner']}: {entry['revision']}")
|
||||||
|
if not report["current_head_entries"]:
|
||||||
|
print(" -")
|
||||||
|
print()
|
||||||
|
print("Release steps:")
|
||||||
|
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
||||||
|
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
||||||
|
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
||||||
|
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
||||||
|
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
"""Tenant, identity, auth, RBAC, and datastore access platform module."""
|
|
||||||
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
"""Authentication and principal helpers for platform access."""
|
|
||||||
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
from govoplan_core.access.permissions.evaluator import scopes_grant
|
|
||||||
|
|
||||||
|
|
||||||
AuthMethod = Literal["session", "api_key", "service_account"]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class Principal:
|
|
||||||
account_id: str
|
|
||||||
membership_id: str | None
|
|
||||||
tenant_id: str | None
|
|
||||||
scopes: frozenset[str]
|
|
||||||
group_ids: frozenset[str]
|
|
||||||
auth_method: AuthMethod
|
|
||||||
api_key_id: str | None = None
|
|
||||||
session_id: str | None = None
|
|
||||||
service_account_id: str | None = None
|
|
||||||
email: str | None = None
|
|
||||||
display_name: str | None = None
|
|
||||||
|
|
||||||
def has(self, required: str) -> bool:
|
|
||||||
return scopes_grant(self.scopes, required)
|
|
||||||
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from govoplan_core.access.db.models import Account, Group, GroupMembership, Membership, Role, RoleBinding
|
|
||||||
from govoplan_core.access.permissions.evaluator import expand_scopes
|
|
||||||
from govoplan_core.core.modules import PermissionDefinition, SubjectType
|
|
||||||
|
|
||||||
|
|
||||||
def membership_group_ids(session: Session, *, tenant_id: str, membership_id: str) -> frozenset[str]:
|
|
||||||
rows = (
|
|
||||||
session.query(Group.id)
|
|
||||||
.join(GroupMembership, GroupMembership.group_id == Group.id)
|
|
||||||
.filter(
|
|
||||||
GroupMembership.tenant_id == tenant_id,
|
|
||||||
GroupMembership.membership_id == membership_id,
|
|
||||||
Group.is_active.is_(True),
|
|
||||||
)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
return frozenset(row[0] for row in rows)
|
|
||||||
|
|
||||||
|
|
||||||
def roles_for_subject(
|
|
||||||
session: Session,
|
|
||||||
*,
|
|
||||||
subject_type: SubjectType,
|
|
||||||
subject_id: str,
|
|
||||||
tenant_id: str | None,
|
|
||||||
) -> list[Role]:
|
|
||||||
query = (
|
|
||||||
session.query(Role)
|
|
||||||
.join(RoleBinding, RoleBinding.role_id == Role.id)
|
|
||||||
.filter(
|
|
||||||
RoleBinding.subject_type == subject_type,
|
|
||||||
RoleBinding.subject_id == subject_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if tenant_id is None:
|
|
||||||
query = query.filter(RoleBinding.tenant_id.is_(None), Role.tenant_id.is_(None))
|
|
||||||
else:
|
|
||||||
query = query.filter(RoleBinding.tenant_id == tenant_id, Role.tenant_id == tenant_id)
|
|
||||||
return query.order_by(Role.name.asc()).all()
|
|
||||||
|
|
||||||
|
|
||||||
def membership_roles(session: Session, membership: Membership) -> list[Role]:
|
|
||||||
roles_by_id = {
|
|
||||||
role.id: role
|
|
||||||
for role in roles_for_subject(
|
|
||||||
session,
|
|
||||||
subject_type="membership",
|
|
||||||
subject_id=membership.id,
|
|
||||||
tenant_id=membership.tenant_id,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
for group_id in membership_group_ids(session, tenant_id=membership.tenant_id, membership_id=membership.id):
|
|
||||||
for role in roles_for_subject(session, subject_type="group", subject_id=group_id, tenant_id=membership.tenant_id):
|
|
||||||
roles_by_id[role.id] = role
|
|
||||||
return list(roles_by_id.values())
|
|
||||||
|
|
||||||
|
|
||||||
def account_system_roles(session: Session, account: Account) -> list[Role]:
|
|
||||||
return roles_for_subject(session, subject_type="account", subject_id=account.id, tenant_id=None)
|
|
||||||
|
|
||||||
|
|
||||||
def principal_scopes(
|
|
||||||
session: Session,
|
|
||||||
*,
|
|
||||||
account: Account,
|
|
||||||
membership: Membership | None,
|
|
||||||
include_system: bool,
|
|
||||||
catalog: dict[str, PermissionDefinition],
|
|
||||||
) -> list[str]:
|
|
||||||
scopes: set[str] = set()
|
|
||||||
if membership is not None:
|
|
||||||
for role in membership_roles(session, membership):
|
|
||||||
scopes.update(role.permissions or [])
|
|
||||||
if include_system:
|
|
||||||
for role in account_system_roles(session, account):
|
|
||||||
scopes.update(role.permissions or [])
|
|
||||||
return expand_scopes(scopes, catalog=catalog)
|
|
||||||
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import secrets
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_RANDOM_BYTES = 32
|
|
||||||
|
|
||||||
|
|
||||||
def generate_secret(prefix: str, *, random_bytes: int = DEFAULT_RANDOM_BYTES) -> str:
|
|
||||||
return prefix + secrets.token_urlsafe(random_bytes)
|
|
||||||
|
|
||||||
|
|
||||||
def hash_secret(secret: str) -> str:
|
|
||||||
return hashlib.sha256(secret.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def verify_secret(secret: str, expected_hash: str) -> bool:
|
|
||||||
return hmac.compare_digest(hash_secret(secret), expected_hash)
|
|
||||||
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
"""Access-platform SQLAlchemy metadata and models."""
|
|
||||||
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from sqlalchemy import JSON, MetaData
|
|
||||||
from sqlalchemy.orm import DeclarativeBase
|
|
||||||
|
|
||||||
from govoplan_core.db.base import NAMING_CONVENTION, TimestampMixin, utcnow
|
|
||||||
|
|
||||||
|
|
||||||
class AccessBase(DeclarativeBase):
|
|
||||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
|
||||||
|
|
||||||
type_annotation_map = {
|
|
||||||
dict[str, Any]: JSON,
|
|
||||||
list[dict[str, Any]]: JSON,
|
|
||||||
list[str]: JSON,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["AccessBase", "NAMING_CONVENTION", "TimestampMixin", "utcnow"]
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user