Compare commits
93 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e11ea81008 | |||
| bc8afeb139 | |||
| f876345656 | |||
| d487726f4d | |||
| e6fc07da37 | |||
| e6d589eb07 | |||
| 59610e21d2 | |||
| cece71d945 | |||
| 22e8183846 | |||
| aa111a5fe1 | |||
| e6062fe9e4 | |||
| 987ca894ed | |||
| 4caa326878 | |||
| 8c4c4456c6 | |||
| 6abe292ac8 | |||
| fea2807754 | |||
| 22646c614c | |||
| 17376332a2 | |||
| 0946bc84a9 | |||
| a18499cbb5 | |||
| 36d7b73bb5 | |||
| b89a2d15f1 | |||
| 7f923afdad | |||
| a7683c5d4a | |||
| 41ad057f7e | |||
| bf0729eb59 | |||
| c4b90181e0 | |||
| 55ed194a99 | |||
| b3b0cf0fca | |||
| fa9119bea7 | |||
| 70ca772138 | |||
| 2eae5c4df6 | |||
| 57fe6c6006 | |||
| 713afdb39b | |||
| 77f8d15d17 | |||
| fda99d40eb | |||
| 5ab1af803b | |||
| 0845e99cf6 | |||
| 28a0a596a6 | |||
| ae74189588 | |||
| 09b5009187 | |||
| 2ca61059dc | |||
| 865901f090 | |||
| 2ac1e64daa | |||
| 7526c5ebb2 | |||
| 8e1f64c790 | |||
| 66e4783d2e | |||
| 7af86b42eb | |||
| ad202f1267 | |||
| 6526f37aae | |||
| 9dabd9356d | |||
| 6502775bf7 | |||
| b2492b820f | |||
| 78d9ae48b2 | |||
| 4cb3e94de3 | |||
| 9131838b98 | |||
| 8e9eb6e1f5 | |||
| 249bf63eb8 | |||
| 248e3dc70e | |||
| 230ecf42b0 | |||
| 825791e9b0 | |||
| 7184b6cdd6 | |||
| ea8c600dce | |||
| 1839693575 | |||
| 183bf7aef0 | |||
| 37a5dfb182 | |||
| 844f934379 | |||
| a98475f7bc | |||
| 1153c9dd36 | |||
| 7eef52776c | |||
| c50ce58ad8 | |||
| 344fc0077f | |||
| 28afc01371 | |||
| 1a29e75db4 | |||
| 57ec960f40 | |||
| 6388afdad8 | |||
| 1a0e90b22d | |||
| 7ad6f6328a | |||
| c79a7124b7 | |||
| abbef5a10b | |||
| 2f559e3f0b | |||
| 9b5418db78 | |||
| 1678602fd6 | |||
| 6e373dcdd6 | |||
| d1c033edc7 | |||
| b5cfba666c | |||
| 78b4afdec4 | |||
| 15596f0742 | |||
| a2320fcb5d | |||
| e6f7c45f0a | |||
| 8aa1943581 | |||
| b9badc9153 | |||
| 9a0c467d55 |
28
.dockerignore
Normal file
28
.dockerignore
Normal file
@@ -0,0 +1,28 @@
|
||||
.git
|
||||
.venv
|
||||
node_modules
|
||||
webui/node_modules
|
||||
audit-reports
|
||||
runtime
|
||||
dist
|
||||
build
|
||||
coverage
|
||||
htmlcov
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.log
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.cache
|
||||
.component-test-build
|
||||
.module-test-build
|
||||
.policy-test-build
|
||||
.template-preview-test-build
|
||||
.import-test-build
|
||||
webui/.component-test-build
|
||||
webui/.module-test-build
|
||||
webui/.policy-test-build
|
||||
webui/.template-preview-test-build
|
||||
webui/.import-test-build
|
||||
32
.env.example
32
.env.example
@@ -1,32 +0,0 @@
|
||||
# GovOPlaN self-hosted install configuration.
|
||||
# Copy to a deployment-local .env or secret store. Do not commit populated secrets.
|
||||
|
||||
APP_ENV=production
|
||||
GOVOPLAN_INSTALL_PROFILE=self-hosted
|
||||
MASTER_KEY_B64=<generate-with-govoplan-config-env-template-generate-secrets>
|
||||
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,docs,ops
|
||||
|
||||
CELERY_ENABLED=true
|
||||
REDIS_URL=redis://127.0.0.1:6379/0
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
|
||||
CORS_ORIGINS=https://govoplan.example.org
|
||||
AUTH_COOKIE_SECURE=true
|
||||
AUTH_COOKIE_SAMESITE=lax
|
||||
AUTH_COOKIE_DOMAIN=
|
||||
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
|
||||
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
|
||||
|
||||
DEV_AUTO_MIGRATE_ENABLED=false
|
||||
DEV_BOOTSTRAP_ENABLED=false
|
||||
DEV_MAILBOX_API_ENABLED=false
|
||||
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable
|
||||
@@ -1,46 +0,0 @@
|
||||
name: Dependency Audit
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
schedule:
|
||||
- cron: "23 3 * * 1"
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Configure SSH for release dependencies
|
||||
env:
|
||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||
chmod 600 ~/.ssh/known_hosts
|
||||
- name: Install backend dev audit dependencies
|
||||
run: |
|
||||
python -m venv .venv
|
||||
.venv/bin/python -m pip install --upgrade pip
|
||||
.venv/bin/python -m pip install -r requirements-release.txt
|
||||
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
|
||||
- name: Install WebUI release dependencies
|
||||
working-directory: webui
|
||||
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||
- name: Run dependency audits
|
||||
run: bash scripts/check-dependency-audits.sh
|
||||
@@ -1,42 +0,0 @@
|
||||
name: Module Matrix
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
module-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Configure SSH for release dependencies
|
||||
env:
|
||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||
chmod 600 ~/.ssh/known_hosts
|
||||
- name: Install backend release dependencies
|
||||
run: |
|
||||
python -m venv .venv
|
||||
.venv/bin/python -m pip install --upgrade pip
|
||||
.venv/bin/python -m pip install -r requirements-release.txt
|
||||
.venv/bin/python -m pip install '.[dev]'
|
||||
- name: Install WebUI release dependencies with test scripts
|
||||
working-directory: webui
|
||||
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||
- name: Run module matrix and contract tests
|
||||
run: bash scripts/check-module-matrix.sh
|
||||
@@ -1,41 +0,0 @@
|
||||
name: Release Integration
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
release-integration:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Configure SSH for release dependencies
|
||||
env:
|
||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||
chmod 600 ~/.ssh/known_hosts
|
||||
- name: Install backend release integration dependencies
|
||||
run: |
|
||||
python -m venv .venv
|
||||
.venv/bin/python -m pip install --upgrade pip
|
||||
.venv/bin/python -m pip install -r requirements-release.txt
|
||||
.venv/bin/python -m pip install '.[dev]'
|
||||
- name: Install WebUI release dependencies
|
||||
working-directory: webui
|
||||
run: bash ../scripts/install-webui-release-dependencies.sh .
|
||||
- name: Run release integration checks
|
||||
run: bash scripts/check-release-integration.sh
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -138,16 +138,21 @@ dist
|
||||
|
||||
# Local WebUI test/build scratch directories
|
||||
.component-test-build/
|
||||
.file-drop-test-build/
|
||||
.module-test-build/
|
||||
.policy-test-build/
|
||||
.template-preview-test-build/
|
||||
.import-test-build/
|
||||
webui/.component-test-build/
|
||||
webui/.file-drop-test-build/
|
||||
webui/.module-test-build/
|
||||
webui/.policy-test-build/
|
||||
webui/.template-preview-test-build/
|
||||
webui/.import-test-build/
|
||||
|
||||
# Security audit reports
|
||||
audit-reports/
|
||||
|
||||
# ---> Python
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
10
AGENTS.md
10
AGENTS.md
@@ -19,9 +19,9 @@ Use targeted commands first:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
```
|
||||
|
||||
For WebUI checks:
|
||||
@@ -36,8 +36,8 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi
|
||||
Run the consolidated focused check when a change touches module discovery, optional integrations, shared mail components, or mailbox listing:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/check-focused.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/check-focused.sh
|
||||
```
|
||||
|
||||
## Working Rules
|
||||
|
||||
59
README.md
59
README.md
@@ -1,6 +1,13 @@
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=module-matrix.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=release-integration.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions?workflow=dependency-audit.yml&actor=0&status=0)
|
||||
# govoplan-core
|
||||
|
||||
<!-- govoplan-repository-type:start -->
|
||||
**Repository type:** system (kernel).
|
||||
<!-- govoplan-repository-type:end -->
|
||||
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=module-matrix.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=release-integration.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=dependency-audit.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=security-audit.yml&actor=0&status=0)
|
||||
|
||||
# govoplan-core
|
||||
|
||||
@@ -38,10 +45,12 @@ composition rules live in core only where they are stable kernel contracts.
|
||||
|
||||
## Backend development
|
||||
|
||||
Create or activate the core virtual environment, then install core and sibling modules from this repository:
|
||||
For whole-product development, create the virtualenv from the meta repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
python3 -m venv .venv
|
||||
./.venv/bin/python -m pip install --upgrade pip
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
@@ -49,7 +58,7 @@ Run the platform server from core through the module-aware development runner. T
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver \
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000
|
||||
```
|
||||
@@ -58,7 +67,7 @@ For example, to test campaign without files or mail:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \
|
||||
ENABLED_MODULES=access,campaigns /mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000
|
||||
```
|
||||
@@ -73,12 +82,29 @@ To run the production-like local profile with PostgreSQL, Redis, a Celery
|
||||
worker, explicit module configuration, and persistent local file storage:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
See [dev/production-like/README.md](dev/production-like/README.md) for ports,
|
||||
environment overrides, and cleanup commands.
|
||||
See `/mnt/DATA/git/govoplan/dev/production-like/README.md` for ports,
|
||||
environment overrides, and cleanup commands. Core keeps wrapper commands during
|
||||
the migration, but whole-product profiles are owned by the meta repository.
|
||||
|
||||
## Security audit
|
||||
|
||||
The repository includes a containerized audit toolbox for SAST, secret scanning,
|
||||
dependency checks, filesystem misconfiguration scans, duplication, and complexity
|
||||
reports. See [SECURITY_AUDIT.md](docs/SECURITY_AUDIT.md) for the operating model.
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||
```
|
||||
|
||||
CI runs the `ci` profile in report-only mode and uploads `audit-reports/` as an
|
||||
artifact. Once the baseline is clean, set `SECURITY_AUDIT_FAIL_ON_FINDINGS=1`
|
||||
or pass `--strict` locally to turn findings into a failing gate.
|
||||
|
||||
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
||||
|
||||
@@ -86,20 +112,23 @@ To verify the effective runtime paths and bootstrap behavior without starting uv
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
```
|
||||
|
||||
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
||||
|
||||
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||
The meta repository owns whole-product `requirements-dev.txt`,
|
||||
`requirements-release.txt`, and the root `.env.example` operator template. Core
|
||||
keeps package metadata and runtime commands. See
|
||||
[RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||
|
||||
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
||||
For self-hosted config bootstrap and validation:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||
```
|
||||
|
||||
## WebUI development
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
path_separator = os
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = sqlite:///./runtime/multimailer-dev.db
|
||||
sqlalchemy.url = sqlite:///./runtime/govoplan-dev.db
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""add audit outbox events to detailed dev track
|
||||
|
||||
Revision ID: 0f1e2d3c4b5a
|
||||
Revises: 4f2a9c8e7b6d
|
||||
Create Date: 2026-07-11 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0f1e2d3c4b5a"
|
||||
down_revision = "4f2a9c8e7b6d"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"audit_outbox_events",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=200), nullable=False),
|
||||
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||
sa.Column("correlation_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("causation_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("classification", sa.String(length=40), nullable=False),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=20), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("dispatched_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_outbox_events")),
|
||||
sa.UniqueConstraint("event_id", name="uq_audit_outbox_events_event_id"),
|
||||
)
|
||||
op.create_index("ix_audit_outbox_events_correlation_id", "audit_outbox_events", ["correlation_id"], unique=False)
|
||||
op.create_index("ix_audit_outbox_events_event_type", "audit_outbox_events", ["event_type"], unique=False)
|
||||
op.create_index(op.f("ix_audit_outbox_events_status"), "audit_outbox_events", ["status"], unique=False)
|
||||
op.create_index(
|
||||
"ix_audit_outbox_events_status_next_attempt_at",
|
||||
"audit_outbox_events",
|
||||
["status", "next_attempt_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("audit_outbox_events")
|
||||
@@ -30,10 +30,14 @@ _TABLE_RENAMES = (
|
||||
("governance_templates", "admin_governance_templates"),
|
||||
("governance_template_assignments", "admin_governance_template_assignments"),
|
||||
)
|
||||
_KNOWN_TABLE_NAMES = {name for pair in _TABLE_RENAMES for name in pair}
|
||||
|
||||
|
||||
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one())
|
||||
if table_name not in _KNOWN_TABLE_NAMES:
|
||||
raise RuntimeError(f"Unexpected table name: {table_name}")
|
||||
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||
|
||||
|
||||
def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None:
|
||||
@@ -19,11 +19,14 @@ LEGACY_SCOPE_TABLE = "tenancy_tenants"
|
||||
CORE_SCOPE_TABLE = "core_scopes"
|
||||
LEGACY_SLUG_INDEX = "ix_tenancy_tenants_slug"
|
||||
CORE_SLUG_INDEX = "ix_core_scopes_slug"
|
||||
_KNOWN_SCOPE_TABLES = {LEGACY_SCOPE_TABLE, CORE_SCOPE_TABLE}
|
||||
|
||||
|
||||
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||
if table_name not in _KNOWN_SCOPE_TABLES:
|
||||
raise RuntimeError(f"Unexpected scope table name: {table_name}")
|
||||
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one())
|
||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||
|
||||
|
||||
def _scope_tables(bind: sa.Connection) -> set[str]:
|
||||
@@ -16,6 +16,7 @@ revision = "9d0e1f2a3b4c"
|
||||
down_revision = "8c9d0e1f2a3b"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
_RECONCILE_CREATE_ALL_TABLES = ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings")
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -28,9 +29,10 @@ def upgrade() -> None:
|
||||
tables = set(inspector.get_table_names())
|
||||
|
||||
# Reconcile only the empty create_all shape for the newly introduced tables.
|
||||
for table_name in ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings"):
|
||||
for table_name in _RECONCILE_CREATE_ALL_TABLES:
|
||||
if table_name in tables:
|
||||
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
||||
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one() # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||
if count:
|
||||
raise RuntimeError(f"Cannot reconcile non-empty create_all table {table_name}")
|
||||
op.drop_table(table_name)
|
||||
@@ -49,7 +49,7 @@ def upgrade() -> None:
|
||||
placeholders = ", ".join(f":action_{index}" for index, _ in enumerate(SYSTEM_ACTIONS))
|
||||
params = {f"action_{index}": action for index, action in enumerate(SYSTEM_ACTIONS)}
|
||||
bind.execute(
|
||||
sa.text(f"UPDATE audit_log SET scope = 'system' WHERE action IN ({placeholders})"),
|
||||
sa.text(f"UPDATE audit_log SET scope = 'system' WHERE action IN ({placeholders})"), # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||
params,
|
||||
)
|
||||
bind.execute(sa.text("UPDATE audit_log SET scope = 'tenant' WHERE scope IS NULL OR scope NOT IN ('tenant', 'system')"))
|
||||
@@ -5,7 +5,11 @@ from logging.config import fileConfig
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata
|
||||
try:
|
||||
from govoplan_access.backend.db import models as access_models # noqa: F401 - populate optional access metadata
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name != "govoplan_access":
|
||||
raise
|
||||
from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata
|
||||
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||
from govoplan_core.core.migrations import migration_metadata_plan
|
||||
@@ -20,7 +24,10 @@ database_url = config.attributes.get("database_url") or settings.database_url
|
||||
config.set_main_option("sqlalchemy.url", database_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
# Migrations can run inside the long-lived application process when module
|
||||
# state changes. Do not let Alembic's logging setup disable loggers that the
|
||||
# server already created (for example slow-request diagnostics).
|
||||
fileConfig(config.config_file_name, disable_existing_loggers=False)
|
||||
|
||||
|
||||
def _target_metadata():
|
||||
|
||||
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
@@ -0,0 +1,892 @@
|
||||
"""v0.1.7 core baseline
|
||||
|
||||
Revision ID: 4f2a9c8e7b6d
|
||||
Revises: None
|
||||
Create Date: 2026-07-11 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '4f2a9c8e7b6d'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _seed_core_defaults() -> None:
|
||||
bind = op.get_bind()
|
||||
now = _now()
|
||||
if not bind.execute(sa.text("SELECT 1 FROM core_system_settings WHERE id = 'global'")).first():
|
||||
settings_table = sa.table(
|
||||
"core_system_settings",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("default_locale", sa.String),
|
||||
sa.column("allow_tenant_custom_groups", sa.Boolean),
|
||||
sa.column("allow_tenant_custom_roles", sa.Boolean),
|
||||
sa.column("allow_tenant_api_keys", sa.Boolean),
|
||||
sa.column("settings", sa.JSON),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
bind.execute(
|
||||
settings_table.insert().values({
|
||||
"id": "global",
|
||||
"default_locale": "en",
|
||||
"allow_tenant_custom_groups": True,
|
||||
"allow_tenant_custom_roles": True,
|
||||
"allow_tenant_api_keys": True,
|
||||
"settings": {},
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}),
|
||||
)
|
||||
system_roles = {
|
||||
"system_owner": {
|
||||
"name": "System owner",
|
||||
"description": "Protected full instance-wide administration.",
|
||||
"permissions": ["system:*"],
|
||||
"is_builtin": True,
|
||||
},
|
||||
"system_admin": {
|
||||
"name": "System administrator",
|
||||
"description": "Manage the instance without granting the protected System owner role.",
|
||||
"permissions": [
|
||||
"system:tenants:read", "system:tenants:create", "system:tenants:update", "system:tenants:suspend",
|
||||
"system:accounts:read", "system:accounts:create", "system:accounts:update", "system:accounts:suspend",
|
||||
"system:roles:read", "system:roles:write", "system:roles:assign", "system:access:read", "system:access:assign",
|
||||
"system:audit:read", "system:settings:read", "system:settings:write", "system:governance:read", "system:governance:write",
|
||||
],
|
||||
"is_builtin": False,
|
||||
},
|
||||
"system_auditor": {
|
||||
"name": "System auditor",
|
||||
"description": "Read-only access to system administration and audit.",
|
||||
"permissions": [
|
||||
"system:tenants:read", "system:accounts:read", "system:roles:read", "system:access:read",
|
||||
"system:audit:read", "system:settings:read", "system:governance:read",
|
||||
],
|
||||
"is_builtin": False,
|
||||
},
|
||||
}
|
||||
roles_table = sa.table(
|
||||
"access_roles",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("tenant_id", sa.String),
|
||||
sa.column("slug", sa.String),
|
||||
sa.column("name", sa.String),
|
||||
sa.column("description", sa.Text),
|
||||
sa.column("permissions", sa.JSON),
|
||||
sa.column("is_builtin", sa.Boolean),
|
||||
sa.column("is_assignable", sa.Boolean),
|
||||
sa.column("system_template_id", sa.String),
|
||||
sa.column("system_required", sa.Boolean),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
for slug, role in system_roles.items():
|
||||
if bind.execute(
|
||||
sa.text("SELECT 1 FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"),
|
||||
{"slug": slug},
|
||||
).first():
|
||||
continue
|
||||
bind.execute(
|
||||
roles_table.insert().values(
|
||||
id=str(uuid4()),
|
||||
tenant_id=None,
|
||||
slug=slug,
|
||||
name=role["name"],
|
||||
description=role["description"],
|
||||
permissions=role["permissions"],
|
||||
is_builtin=bool(role["is_builtin"]),
|
||||
is_assignable=True,
|
||||
system_template_id=None,
|
||||
system_required=False,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('core_scopes',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('allow_custom_groups', sa.Boolean(), nullable=True),
|
||||
sa.Column('allow_custom_roles', sa.Boolean(), nullable=True),
|
||||
sa.Column('allow_api_keys', sa.Boolean(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_scopes'))
|
||||
)
|
||||
op.create_index(op.f('ix_core_scopes_slug'), 'core_scopes', ['slug'], unique=True)
|
||||
op.create_table('access_accounts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('email', sa.String(length=320), nullable=False),
|
||||
sa.Column('normalized_email', sa.String(length=320), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||
sa.Column('password_reset_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_accounts'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_accounts_normalized_email'), 'access_accounts', ['normalized_email'], unique=True)
|
||||
op.create_table('access_groups',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_groups')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_access_groups_system_template_id'), 'access_groups', ['system_template_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False)
|
||||
op.create_table('access_roles',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||
sa.Column('is_builtin', sa.Boolean(), nullable=False),
|
||||
sa.Column('is_assignable', sa.Boolean(), nullable=False),
|
||||
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_roles')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_access_roles_system_template_id'), 'access_roles', ['system_template_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False)
|
||||
op.create_index('uq_roles_system_slug', 'access_roles', ['slug'], unique=True, sqlite_where=sa.text('tenant_id IS NULL'), postgresql_where=sa.text('tenant_id IS NULL'))
|
||||
op.create_table('admin_governance_templates',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('kind', sa.String(length=20), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_templates')),
|
||||
sa.UniqueConstraint('kind', 'slug', name='uq_governance_templates_kind_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_admin_governance_templates_kind'), 'admin_governance_templates', ['kind'], unique=False)
|
||||
op.create_table('attachment_blobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('mime_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('storage_bucket', sa.String(length=255), nullable=False),
|
||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
||||
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
||||
)
|
||||
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
||||
op.create_table('audit_outbox_events',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('event_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('event_type', sa.String(length=200), nullable=False),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('correlation_id', sa.String(length=128), nullable=True),
|
||||
sa.Column('causation_id', sa.String(length=128), nullable=True),
|
||||
sa.Column('classification', sa.String(length=40), nullable=False),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('status', sa.String(length=20), nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
||||
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('dispatched_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_outbox_events')),
|
||||
sa.UniqueConstraint('event_id', name='uq_audit_outbox_events_event_id')
|
||||
)
|
||||
op.create_index('ix_audit_outbox_events_correlation_id', 'audit_outbox_events', ['correlation_id'], unique=False)
|
||||
op.create_index('ix_audit_outbox_events_event_type', 'audit_outbox_events', ['event_type'], unique=False)
|
||||
op.create_index(op.f('ix_audit_outbox_events_status'), 'audit_outbox_events', ['status'], unique=False)
|
||||
op.create_index('ix_audit_outbox_events_status_next_attempt_at', 'audit_outbox_events', ['status', 'next_attempt_at'], unique=False)
|
||||
op.create_table('core_change_sequence',
|
||||
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||
sa.Column('resource_type', sa.String(length=100), nullable=False),
|
||||
sa.Column('resource_id', sa.String(length=255), nullable=False),
|
||||
sa.Column('operation', sa.String(length=30), nullable=False),
|
||||
sa.Column('actor_type', sa.String(length=30), nullable=True),
|
||||
sa.Column('actor_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence'))
|
||||
)
|
||||
op.create_index(op.f('ix_core_change_sequence_actor_id'), 'core_change_sequence', ['actor_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_actor_type'), 'core_change_sequence', ['actor_type'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_collection'), 'core_change_sequence', ['collection'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_collection_id', 'core_change_sequence', ['collection', 'id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_created_at'), 'core_change_sequence', ['created_at'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_module_id'), 'core_change_sequence', ['module_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_operation'), 'core_change_sequence', ['operation'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_resource', 'core_change_sequence', ['module_id', 'resource_type', 'resource_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_resource_id'), 'core_change_sequence', ['resource_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_resource_type'), 'core_change_sequence', ['resource_type'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_tenant_id'), 'core_change_sequence', ['tenant_id'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_tenant_module_id', 'core_change_sequence', ['tenant_id', 'module_id', 'id'], unique=False)
|
||||
op.create_table('core_change_sequence_retention_floor',
|
||||
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||
sa.Column('tenant_key', sa.String(length=36), nullable=False),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||
sa.Column('min_valid_sequence', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence_retention_floor')),
|
||||
sa.UniqueConstraint('tenant_key', 'module_id', 'collection', name='uq_core_change_sequence_retention_scope')
|
||||
)
|
||||
op.create_index('ix_core_change_sequence_retention_scope', 'core_change_sequence_retention_floor', ['tenant_key', 'module_id', 'collection'], unique=False)
|
||||
op.create_table('core_system_settings',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||
sa.Column('allow_tenant_custom_groups', sa.Boolean(), nullable=False),
|
||||
sa.Column('allow_tenant_custom_roles', sa.Boolean(), nullable=False),
|
||||
sa.Column('allow_tenant_api_keys', sa.Boolean(), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_system_settings'))
|
||||
)
|
||||
op.create_table('file_blobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('storage_backend', sa.String(length=50), nullable=False),
|
||||
sa.Column('storage_bucket', sa.String(length=255), nullable=True),
|
||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('ref_count', sa.Integer(), nullable=False),
|
||||
sa.Column('retained_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_blobs')),
|
||||
sa.UniqueConstraint('tenant_id', 'checksum_sha256', 'size_bytes', name='uq_file_blobs_tenant_checksum_size')
|
||||
)
|
||||
op.create_index(op.f('ix_file_blobs_checksum_sha256'), 'file_blobs', ['checksum_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_file_blobs_tenant_id'), 'file_blobs', ['tenant_id'], unique=False)
|
||||
op.create_table('access_group_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_group_role_assignments_group_id_access_groups'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_group_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_group_role_assignments')),
|
||||
sa.UniqueConstraint('tenant_id', 'group_id', 'role_id', name='uq_group_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_group_id'), 'access_group_role_assignments', ['group_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_role_id'), 'access_group_role_assignments', ['role_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_tenant_id'), 'access_group_role_assignments', ['tenant_id'], unique=False)
|
||||
op.create_table('access_system_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_system_role_assignments_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_system_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_system_role_assignments')),
|
||||
sa.UniqueConstraint('account_id', 'role_id', name='uq_system_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_system_role_assignments_account_id'), 'access_system_role_assignments', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_system_role_assignments_role_id'), 'access_system_role_assignments', ['role_id'], unique=False)
|
||||
op.create_table('access_users',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('email', sa.String(length=320), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
||||
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_users_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_users')),
|
||||
sa.UniqueConstraint('tenant_id', 'account_id', name='uq_users_tenant_account'),
|
||||
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
||||
)
|
||||
op.create_index(op.f('ix_access_users_account_id'), 'access_users', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False)
|
||||
op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False)
|
||||
op.create_table('admin_governance_template_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('template_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('mode', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['template_id'], ['admin_governance_templates.id'], name=op.f('fk_admin_governance_template_assignments_template_id_admin_governance_templates'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_template_assignments')),
|
||||
sa.UniqueConstraint('template_id', 'tenant_id', name='uq_governance_template_tenant')
|
||||
)
|
||||
op.create_index(op.f('ix_admin_governance_template_assignments_template_id'), 'admin_governance_template_assignments', ['template_id'], unique=False)
|
||||
op.create_index(op.f('ix_admin_governance_template_assignments_tenant_id'), 'admin_governance_template_assignments', ['tenant_id'], unique=False)
|
||||
op.create_table('access_api_keys',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('prefix', sa.String(length=16), nullable=False),
|
||||
sa.Column('key_hash', sa.String(length=128), nullable=False),
|
||||
sa.Column('scopes', sa.JSON(), nullable=False),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_api_keys_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_api_keys'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False)
|
||||
op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False)
|
||||
op.create_table('access_auth_sessions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('token_hash', sa.String(length=128), nullable=False),
|
||||
sa.Column('csrf_token_hash', sa.String(length=128), nullable=True),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('last_seen_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('user_agent', sa.String(length=500), nullable=True),
|
||||
sa.Column('ip_address', sa.String(length=100), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_auth_sessions_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_auth_sessions_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_auth_sessions'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_auth_sessions_account_id'), 'access_auth_sessions', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_expires_at'), 'access_auth_sessions', ['expires_at'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_revoked_at'), 'access_auth_sessions', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_tenant_id'), 'access_auth_sessions', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_token_hash'), 'access_auth_sessions', ['token_hash'], unique=True)
|
||||
op.create_index(op.f('ix_access_auth_sessions_user_id'), 'access_auth_sessions', ['user_id'], unique=False)
|
||||
op.create_table('access_user_group_memberships',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_user_group_memberships_group_id_access_groups'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_group_memberships_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_group_memberships')),
|
||||
sa.UniqueConstraint('tenant_id', 'user_id', 'group_id', name='uq_user_group_memberships')
|
||||
)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_group_id'), 'access_user_group_memberships', ['group_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_tenant_id'), 'access_user_group_memberships', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_user_id'), 'access_user_group_memberships', ['user_id'], unique=False)
|
||||
op.create_table('access_user_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_user_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_role_assignments_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_role_assignments')),
|
||||
sa.UniqueConstraint('tenant_id', 'user_id', 'role_id', name='uq_user_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_role_id'), 'access_user_role_assignments', ['role_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_tenant_id'), 'access_user_role_assignments', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_user_id'), 'access_user_role_assignments', ['user_id'], unique=False)
|
||||
op.create_table('campaigns',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('external_id', sa.String(length=255), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_campaigns_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_campaigns_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
||||
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
||||
)
|
||||
op.create_index(op.f('ix_campaigns_created_by_user_id'), 'campaigns', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_external_id'), 'campaigns', ['external_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_owner_group_id'), 'campaigns', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_owner_user_id'), 'campaigns', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_status'), 'campaigns', ['status'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_tenant_id'), 'campaigns', ['tenant_id'], unique=False)
|
||||
op.create_table('file_assets',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('display_path', sa.String(length=1000), nullable=False),
|
||||
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_assets_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_assets_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_assets_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_assets'))
|
||||
)
|
||||
op.create_index(op.f('ix_file_assets_created_by_user_id'), 'file_assets', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_current_version_id'), 'file_assets', ['current_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_deleted_at'), 'file_assets', ['deleted_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_display_path'), 'file_assets', ['display_path'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_filename'), 'file_assets', ['filename'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_group_id'), 'file_assets', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_type'), 'file_assets', ['owner_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_user_id'), 'file_assets', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_tenant_id'), 'file_assets', ['tenant_id'], unique=False)
|
||||
op.create_table('file_folders',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('path', sa.String(length=1000), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_folders_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_folders_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_folders_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_folders'))
|
||||
)
|
||||
op.create_index(op.f('ix_file_folders_created_by_user_id'), 'file_folders', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_deleted_at'), 'file_folders', ['deleted_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_group_id'), 'file_folders', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_type'), 'file_folders', ['owner_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_user_id'), 'file_folders', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_path'), 'file_folders', ['path'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_tenant_id'), 'file_folders', ['tenant_id'], unique=False)
|
||||
op.create_index('uq_file_folders_active_group_path', 'file_folders', ['tenant_id', 'owner_group_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"))
|
||||
op.create_index('uq_file_folders_active_user_path', 'file_folders', ['tenant_id', 'owner_user_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"))
|
||||
op.create_table('mail_server_profiles',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('scope_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('scope_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('smtp_config', sa.JSON(), nullable=False),
|
||||
sa.Column('smtp_username', sa.String(length=320), nullable=True),
|
||||
sa.Column('smtp_password_encrypted', sa.Text(), nullable=True),
|
||||
sa.Column('imap_config', sa.JSON(), nullable=True),
|
||||
sa.Column('imap_username', sa.String(length=320), nullable=True),
|
||||
sa.Column('imap_password_encrypted', sa.Text(), nullable=True),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('updated_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['updated_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_updated_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_mail_server_profiles')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_mail_server_profiles_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_mail_server_profiles_created_by_user_id'), 'mail_server_profiles', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_is_active'), 'mail_server_profiles', ['is_active'], unique=False)
|
||||
op.create_index('ix_mail_server_profiles_scope', 'mail_server_profiles', ['scope_type', 'scope_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_scope_id'), 'mail_server_profiles', ['scope_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_scope_type'), 'mail_server_profiles', ['scope_type'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_tenant_id'), 'mail_server_profiles', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_updated_by_user_id'), 'mail_server_profiles', ['updated_by_user_id'], unique=False)
|
||||
op.create_table('attachment_instances',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('logical_name', sa.String(length=500), nullable=True),
|
||||
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||
sa.Column('tags', sa.JSON(), nullable=False),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
||||
)
|
||||
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_campaign_id'), 'attachment_instances', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_owner_user_id'), 'attachment_instances', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_tenant_id'), 'attachment_instances', ['tenant_id'], unique=False)
|
||||
op.create_table('audit_log',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('scope', sa.String(length=20), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('api_key_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('action', sa.String(length=100), nullable=False),
|
||||
sa.Column('object_type', sa.String(length=100), nullable=True),
|
||||
sa.Column('object_id', sa.String(length=100), nullable=True),
|
||||
sa.Column('details', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_access_api_keys'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
||||
)
|
||||
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_api_key_id'), 'audit_log', ['api_key_id'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_object_id'), 'audit_log', ['object_id'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_object_type'), 'audit_log', ['object_type'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_scope'), 'audit_log', ['scope'], unique=False)
|
||||
op.create_index('ix_audit_log_scope_created_at', 'audit_log', ['scope', 'created_at'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_tenant_id'), 'audit_log', ['tenant_id'], unique=False)
|
||||
op.create_index('ix_audit_log_tenant_scope_created_at', 'audit_log', ['tenant_id', 'scope', 'created_at'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_user_id'), 'audit_log', ['user_id'], unique=False)
|
||||
op.create_table('campaign_shares',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_shares_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_shares')),
|
||||
sa.UniqueConstraint('campaign_id', 'target_type', 'target_id', name='uq_campaign_share_target')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_shares_campaign_id'), 'campaign_shares', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_created_by_user_id'), 'campaign_shares', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_revoked_at'), 'campaign_shares', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_target_id'), 'campaign_shares', ['target_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_target_type'), 'campaign_shares', ['target_type'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_tenant_id'), 'campaign_shares', ['tenant_id'], unique=False)
|
||||
op.create_table('campaign_versions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||
sa.Column('raw_json', sa.JSON(), nullable=False),
|
||||
sa.Column('schema_version', sa.String(length=50), nullable=False),
|
||||
sa.Column('source_filename', sa.String(length=500), nullable=True),
|
||||
sa.Column('source_base_path', sa.String(length=1000), nullable=True),
|
||||
sa.Column('workflow_state', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_flow', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_step', sa.String(length=100), nullable=True),
|
||||
sa.Column('is_complete', sa.Boolean(), nullable=False),
|
||||
sa.Column('editor_state', sa.JSON(), nullable=False),
|
||||
sa.Column('autosaved_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('locked_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('user_lock_state', sa.String(length=20), nullable=True),
|
||||
sa.Column('user_locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('user_locked_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('validation_summary', sa.JSON(), nullable=True),
|
||||
sa.Column('build_summary', sa.JSON(), nullable=True),
|
||||
sa.Column('execution_snapshot', sa.JSON(), nullable=True),
|
||||
sa.Column('execution_snapshot_hash', sa.String(length=64), nullable=True),
|
||||
sa.Column('execution_snapshot_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_versions_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['user_locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_user_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_versions')),
|
||||
sa.UniqueConstraint('campaign_id', 'version_number', name='uq_campaign_versions_campaign_number')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_versions_campaign_id'), 'campaign_versions', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_current_flow'), 'campaign_versions', ['current_flow'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_execution_snapshot_hash'), 'campaign_versions', ['execution_snapshot_hash'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_locked_by_user_id'), 'campaign_versions', ['locked_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_user_lock_state'), 'campaign_versions', ['user_lock_state'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_user_locked_by_user_id'), 'campaign_versions', ['user_locked_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_workflow_state'), 'campaign_versions', ['workflow_state'], unique=False)
|
||||
op.create_table('file_shares',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_shares_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_shares')),
|
||||
sa.UniqueConstraint('file_asset_id', 'target_type', 'target_id', 'revoked_at', name='uq_file_shares_active_target')
|
||||
)
|
||||
op.create_index(op.f('ix_file_shares_created_by_user_id'), 'file_shares', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_file_asset_id'), 'file_shares', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_revoked_at'), 'file_shares', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_target_id'), 'file_shares', ['target_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_target_type'), 'file_shares', ['target_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_tenant_id'), 'file_shares', ['tenant_id'], unique=False)
|
||||
op.create_table('file_versions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||
sa.Column('filename_at_upload', sa.String(length=500), nullable=False),
|
||||
sa.Column('display_path_at_upload', sa.String(length=1000), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['file_blobs.id'], name=op.f('fk_file_versions_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_versions_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_versions_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_versions')),
|
||||
sa.UniqueConstraint('file_asset_id', 'version_number', name='uq_file_versions_asset_number')
|
||||
)
|
||||
op.create_index(op.f('ix_file_versions_blob_id'), 'file_versions', ['blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_checksum_sha256'), 'file_versions', ['checksum_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_created_by_user_id'), 'file_versions', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_file_asset_id'), 'file_versions', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_tenant_id'), 'file_versions', ['tenant_id'], unique=False)
|
||||
op.create_table('campaign_jobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('entry_index', sa.Integer(), nullable=False),
|
||||
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('recipient_email', sa.String(length=320), nullable=True),
|
||||
sa.Column('subject', sa.String(length=998), nullable=True),
|
||||
sa.Column('message_id_header', sa.String(length=255), nullable=True),
|
||||
sa.Column('eml_storage_key', sa.String(length=1000), nullable=True),
|
||||
sa.Column('eml_local_path', sa.String(length=1000), nullable=True),
|
||||
sa.Column('eml_size_bytes', sa.Integer(), nullable=True),
|
||||
sa.Column('eml_sha256', sa.String(length=64), nullable=True),
|
||||
sa.Column('build_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('validation_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('queue_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('send_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('imap_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('attempt_count', sa.Integer(), nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('queued_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claimed_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||
sa.Column('smtp_started_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('outcome_unknown_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('sent_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('resolved_recipients', sa.JSON(), nullable=True),
|
||||
sa.Column('resolved_attachments', sa.JSON(), nullable=False),
|
||||
sa.Column('issues_snapshot', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
||||
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_jobs_build_status'), 'campaign_jobs', ['build_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_campaign_id'), 'campaign_jobs', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_campaign_version_id'), 'campaign_jobs', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_claim_token'), 'campaign_jobs', ['claim_token'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_eml_sha256'), 'campaign_jobs', ['eml_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_entry_id'), 'campaign_jobs', ['entry_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_imap_status'), 'campaign_jobs', ['imap_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_queue_status'), 'campaign_jobs', ['queue_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_recipient_email'), 'campaign_jobs', ['recipient_email'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_send_status'), 'campaign_jobs', ['send_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_tenant_id'), 'campaign_jobs', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_validation_status'), 'campaign_jobs', ['validation_status'], unique=False)
|
||||
op.create_table('campaign_attachment_uses',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_job_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('entry_index', sa.Integer(), nullable=True),
|
||||
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('filename_used', sa.String(length=500), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('use_stage', sa.String(length=20), nullable=False),
|
||||
sa.Column('used_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_attachment_uses_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_attachment_uses_campaign_job_id_campaign_jobs'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_attachment_uses_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_campaign_attachment_uses_file_asset_id_file_assets'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['file_blob_id'], ['file_blobs.id'], name=op.f('fk_campaign_attachment_uses_file_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['file_version_id'], ['file_versions.id'], name=op.f('fk_campaign_attachment_uses_file_version_id_file_versions'), ondelete='RESTRICT'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_attachment_uses')),
|
||||
sa.UniqueConstraint('campaign_job_id', 'file_version_id', 'filename_used', 'use_stage', name='uq_campaign_attachment_uses_job_file_stage')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_id'), 'campaign_attachment_uses', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_job_id'), 'campaign_attachment_uses', ['campaign_job_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_version_id'), 'campaign_attachment_uses', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_entry_id'), 'campaign_attachment_uses', ['entry_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_asset_id'), 'campaign_attachment_uses', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_blob_id'), 'campaign_attachment_uses', ['file_blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_version_id'), 'campaign_attachment_uses', ['file_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_tenant_id'), 'campaign_attachment_uses', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_use_stage'), 'campaign_attachment_uses', ['use_stage'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_used_at'), 'campaign_attachment_uses', ['used_at'], unique=False)
|
||||
op.create_table('campaign_issues',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('severity', sa.String(length=20), nullable=False),
|
||||
sa.Column('code', sa.String(length=100), nullable=False),
|
||||
sa.Column('message', sa.Text(), nullable=False),
|
||||
sa.Column('source', sa.String(length=255), nullable=True),
|
||||
sa.Column('behavior', sa.String(length=50), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_campaign_version_id'), 'campaign_issues', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_code'), 'campaign_issues', ['code'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_job_id'), 'campaign_issues', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_severity'), 'campaign_issues', ['severity'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_tenant_id'), 'campaign_issues', ['tenant_id'], unique=False)
|
||||
op.create_table('imap_append_attempts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||
sa.Column('folder', sa.String(length=500), nullable=True),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_imap_append_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_imap_append_attempts'))
|
||||
)
|
||||
op.create_index(op.f('ix_imap_append_attempts_job_id'), 'imap_append_attempts', ['job_id'], unique=False)
|
||||
op.create_table('send_attempts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||
sa.Column('smtp_status_code', sa.Integer(), nullable=True),
|
||||
sa.Column('smtp_response', sa.Text(), nullable=True),
|
||||
sa.Column('error_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_send_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_send_attempts'))
|
||||
)
|
||||
op.create_index(op.f('ix_send_attempts_claim_token'), 'send_attempts', ['claim_token'], unique=False)
|
||||
op.create_index(op.f('ix_send_attempts_job_id'), 'send_attempts', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_send_attempts_status'), 'send_attempts', ['status'], unique=False)
|
||||
_seed_core_defaults()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table('send_attempts')
|
||||
op.drop_table('imap_append_attempts')
|
||||
op.drop_table('campaign_issues')
|
||||
op.drop_table('campaign_attachment_uses')
|
||||
op.drop_table('campaign_jobs')
|
||||
op.drop_table('file_versions')
|
||||
op.drop_table('file_shares')
|
||||
op.drop_table('campaign_versions')
|
||||
op.drop_table('campaign_shares')
|
||||
op.drop_table('audit_log')
|
||||
op.drop_table('attachment_instances')
|
||||
op.drop_table('mail_server_profiles')
|
||||
op.drop_table('file_folders')
|
||||
op.drop_table('file_assets')
|
||||
op.drop_table('campaigns')
|
||||
op.drop_table('access_user_role_assignments')
|
||||
op.drop_table('access_user_group_memberships')
|
||||
op.drop_table('access_auth_sessions')
|
||||
op.drop_table('access_api_keys')
|
||||
op.drop_table('admin_governance_template_assignments')
|
||||
op.drop_table('access_users')
|
||||
op.drop_table('access_system_role_assignments')
|
||||
op.drop_table('access_group_role_assignments')
|
||||
op.drop_table('file_blobs')
|
||||
op.drop_table('core_system_settings')
|
||||
op.drop_table('core_change_sequence_retention_floor')
|
||||
op.drop_table('core_change_sequence')
|
||||
op.drop_table('audit_outbox_events')
|
||||
op.drop_table('attachment_blobs')
|
||||
op.drop_table('admin_governance_templates')
|
||||
op.drop_table('access_roles')
|
||||
op.drop_table('access_groups')
|
||||
op.drop_table('access_accounts')
|
||||
op.drop_table('core_scopes')
|
||||
@@ -1,5 +0,0 @@
|
||||
GOVOPLAN_POSTGRES_DB=govoplan
|
||||
GOVOPLAN_POSTGRES_USER=govoplan
|
||||
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
|
||||
GOVOPLAN_POSTGRES_PORT=55432
|
||||
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan
|
||||
@@ -1,126 +0,0 @@
|
||||
# PostgreSQL Development Profile
|
||||
|
||||
GovOPlaN development now defaults to PostgreSQL:
|
||||
|
||||
```text
|
||||
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||
```
|
||||
|
||||
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
|
||||
|
||||
```text
|
||||
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||
```
|
||||
|
||||
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
|
||||
The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if
|
||||
you need a different database.
|
||||
|
||||
## Host PostgreSQL Setup
|
||||
|
||||
Create the default local role and database from a host shell:
|
||||
|
||||
```bash
|
||||
export GOVOPLAN_PG_DB=govoplan_dev
|
||||
export GOVOPLAN_PG_USER=govoplan_dev
|
||||
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
|
||||
echo
|
||||
|
||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
|
||||
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
|
||||
else
|
||||
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
|
||||
fi
|
||||
|
||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
|
||||
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
|
||||
fi
|
||||
|
||||
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
|
||||
|
||||
touch ~/.pgpass
|
||||
chmod 600 ~/.pgpass
|
||||
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
|
||||
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
|
||||
mv ~/.pgpass.tmp ~/.pgpass
|
||||
chmod 600 ~/.pgpass
|
||||
```
|
||||
|
||||
Check access:
|
||||
|
||||
```bash
|
||||
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
|
||||
-c 'select current_user, current_database();'
|
||||
```
|
||||
|
||||
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
|
||||
available through:
|
||||
|
||||
```bash
|
||||
flatpak-spawn --host /usr/bin/psql --version
|
||||
flatpak-spawn --host /usr/bin/pg_dump --version
|
||||
flatpak-spawn --host /usr/bin/pg_restore --version
|
||||
```
|
||||
|
||||
## Run GovOPlaN Against Host PostgreSQL
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||
--with-dev-data
|
||||
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
```
|
||||
|
||||
For the full backend and WebUI launcher:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-dev.sh
|
||||
```
|
||||
|
||||
To force the old SQLite fallback for a disposable local run:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh
|
||||
```
|
||||
|
||||
## Disposable Docker Testbed
|
||||
|
||||
This testbed is for migration and module-permutation checks. It is not a
|
||||
production deployment profile.
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
```
|
||||
|
||||
Run the integration check from the core checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
|
||||
`--reset-schema` drops and recreates the `public` schema before every module
|
||||
set. Use it only against this disposable database.
|
||||
|
||||
Stop the testbed:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Remove all test data:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
@@ -1,22 +0,0 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: govoplan-core-postgres-dev
|
||||
environment:
|
||||
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
|
||||
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
|
||||
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
@@ -1,27 +0,0 @@
|
||||
APP_ENV=staging
|
||||
GOVOPLAN_INSTALL_PROFILE=production-like
|
||||
MASTER_KEY_B64=
|
||||
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
|
||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
|
||||
|
||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
|
||||
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
REDIS_URL=redis://127.0.0.1:56379/0
|
||||
CELERY_ENABLED=true
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||
AUTH_COOKIE_SECURE=false
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||
DEV_AUTO_MIGRATE_ENABLED=false
|
||||
DEV_BOOTSTRAP_ENABLED=true
|
||||
@@ -1,60 +0,0 @@
|
||||
# Production-Like Development Profile
|
||||
|
||||
This profile runs the shared services that production depends on while keeping
|
||||
API, worker, and WebUI code in the editable local repositories.
|
||||
|
||||
It provides:
|
||||
|
||||
- PostgreSQL with a persistent Docker volume
|
||||
- Redis with append-only persistence
|
||||
- explicit `ENABLED_MODULES`
|
||||
- local durable file storage under `runtime/production-like/files`
|
||||
- a Celery worker process using the same queues as the API
|
||||
|
||||
Start it from the core repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
The helper wrapper exposes repeatable lifecycle commands:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
scripts/production-like-dev.sh stop
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The launcher uses `dev/production-like/.env` when present, otherwise
|
||||
`dev/production-like/.env.example`. Copy the example when you want local port or
|
||||
password changes:
|
||||
|
||||
```bash
|
||||
cp dev/production-like/.env.example dev/production-like/.env
|
||||
```
|
||||
|
||||
The API and worker use:
|
||||
|
||||
```text
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
REDIS_URL=redis://127.0.0.1:56379/0
|
||||
CELERY_ENABLED=true
|
||||
```
|
||||
|
||||
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
|
||||
containers keep running by default so the next launch is fast. To stop them too:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
To remove all profile data:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
```
|
||||
@@ -1,37 +0,0 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: govoplan-production-like-postgres
|
||||
environment:
|
||||
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
|
||||
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
|
||||
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: govoplan-production-like-redis
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
@@ -58,18 +58,18 @@ Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.
|
||||
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/check-focused.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/check-focused.sh
|
||||
```
|
||||
|
||||
For smaller changes, prefer the narrow command named in the relevant `AGENTS.md` file. Examples:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||
|
||||
cd /mnt/DATA/git/govoplan-mail
|
||||
/mnt/DATA/git/govoplan-core/.venv/bin/python -m unittest discover -s tests
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest discover -s tests
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core/webui
|
||||
PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations
|
||||
|
||||
@@ -6,23 +6,24 @@ metadata and can fail for newly disclosed advisories without a source change.
|
||||
|
||||
## Local Workflow
|
||||
|
||||
Install the development audit dependency once:
|
||||
Install the whole-product development dependencies once from the meta
|
||||
repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Run both backend and WebUI production audits:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
The script runs:
|
||||
|
||||
- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||
- `tools/checks/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||
legacy editable package metadata, deprecated framework constants, and the
|
||||
Starlette `TestClient` deprecation smoke when test dependencies are present
|
||||
- `python -m pip_audit --progress-spinner off`
|
||||
@@ -31,11 +32,12 @@ The script runs:
|
||||
For fast local checks without vulnerability metadata lookups, run:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 \
|
||||
bash tools/checks/check-dependency-hygiene.sh
|
||||
```
|
||||
|
||||
This is also part of `scripts/check-focused.sh`, so resolver drift and
|
||||
This is also part of `govoplan/tools/checks/check-focused.sh`, so resolver drift and
|
||||
deprecation regressions fail close to the code change that introduced them.
|
||||
|
||||
Override tool paths when testing from a disposable environment:
|
||||
@@ -43,12 +45,13 @@ Override tool paths when testing from a disposable environment:
|
||||
```bash
|
||||
PYTHON=/tmp/govoplan-audit/bin/python \
|
||||
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
||||
bash scripts/check-dependency-audits.sh
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core \
|
||||
bash /mnt/DATA/git/govoplan/tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
## CI Workflow
|
||||
|
||||
`.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||
`govoplan/.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
||||
pull requests, and a weekly schedule.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ installer CLI/daemon for package mutation under maintenance mode.
|
||||
Generate a deployment-local template:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||
--profile self-hosted \
|
||||
--generate-secrets \
|
||||
@@ -54,8 +54,9 @@ PY
|
||||
| Setting | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
||||
| `GOVOPLAN_MIGRATION_TRACK` | `release` | Use the release track for normal runtime and deployments. Use `dev` only for fresh/disposable databases that intentionally replay detailed development migrations. |
|
||||
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
||||
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `govoplan/tools/launch/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||
|
||||
Operator rule: take a database backup before applying migrations or destructive
|
||||
module retirement. For non-SQLite databases, configure deployment-specific
|
||||
@@ -68,10 +69,11 @@ supported only for tiny disposable profiles and unit-test style smoke runs.
|
||||
Production/staging deployments should use a managed PostgreSQL database and
|
||||
explicit migration commands.
|
||||
|
||||
Install the server extra so the `psycopg` driver is available:
|
||||
Install the full release profile from the meta repository so core, modules, and
|
||||
the `psycopg` driver are available:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
@@ -113,37 +115,41 @@ pg_restore --clean --if-exists \
|
||||
```
|
||||
|
||||
For local development, create the host database described in
|
||||
`dev/postgres/README.md`, then run:
|
||||
`/mnt/DATA/git/govoplan/dev/postgres/README.md`, then run:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||
--with-dev-data
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
scripts/launch-dev.sh
|
||||
tools/launch/launch-dev.sh
|
||||
```
|
||||
|
||||
For disposable local validation against a throwaway PostgreSQL instance, use
|
||||
the bundled PostgreSQL testbed:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
tools/checks/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
|
||||
The integration check runs migrations and startup smoke checks across the
|
||||
standard module permutations. `--reset-schema` is destructive and belongs only
|
||||
on throwaway databases.
|
||||
standard module permutations. It first requires the retirement atomicity proof,
|
||||
using Files' real secret-owning provider and Audit's persistent recorder. That
|
||||
proof uses only random, test-owned schemas and cleans them afterward; it does
|
||||
not reset `public`. `--reset-schema` is destructive and belongs only on
|
||||
throwaway databases. Do not pass `--skip-retirement-atomicity` when collecting
|
||||
release evidence.
|
||||
|
||||
### Broker And Workers
|
||||
|
||||
@@ -151,16 +157,24 @@ on throwaway databases.
|
||||
| --- | --- | --- |
|
||||
| `REDIS_URL` | `redis://redis:6379/0` | Celery broker/result backend when async workers are enabled. |
|
||||
| `CELERY_ENABLED` | `false` | Local/dev can send synchronously. Production campaign delivery should run workers and set this to `true`. |
|
||||
| `CELERY_QUEUES` | `send_email,append_sent,default` | Queue list expected by worker/process manager definitions. |
|
||||
| `CELERY_QUEUES` | `send_email,append_sent,notifications,calendar,default` | Queue list expected by worker/process manager definitions. The Calendar queue drains durable external-calendar operations. |
|
||||
|
||||
Worker command:
|
||||
|
||||
```bash
|
||||
python -m celery -A govoplan_core.celery_app:celery worker \
|
||||
--queues send_email,append_sent,default \
|
||||
--queues send_email,append_sent,notifications,calendar,default \
|
||||
--loglevel INFO
|
||||
```
|
||||
|
||||
Run Celery beat as a separately supervised process. Its built-in one-minute
|
||||
schedule recovers Calendar outbox rows left behind by broker failures, process
|
||||
crashes, and expired worker leases:
|
||||
|
||||
```bash
|
||||
python -m celery -A govoplan_core.celery_app:celery beat --loglevel INFO
|
||||
```
|
||||
|
||||
### Storage
|
||||
|
||||
| Setting | Default | Notes |
|
||||
@@ -182,15 +196,60 @@ prefer `FILE_STORAGE_*`.
|
||||
| Setting | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `CORS_ORIGINS` | local dev origins | Set to the exact WebUI origins in staging/production. |
|
||||
| `GOVOPLAN_TRUSTED_HOSTS` | empty | Exact API host names accepted by the application. Production-like validation requires an explicit list; narrowly scoped `*.example.org` entries are supported. |
|
||||
| `FORWARDED_ALLOW_IPS` | Uvicorn default | Address or network of the trusted reverse proxy. Never use `*` in production-like deployments. |
|
||||
| `AUTH_SESSION_COOKIE_NAME` | configured default | Change only through a controlled rollout because it logs users out. |
|
||||
| `AUTH_CSRF_COOKIE_NAME` | configured default | Must match WebUI/API deployment. |
|
||||
| `AUTH_COOKIE_SECURE` | `false` | Set `true` behind HTTPS. |
|
||||
| `AUTH_COOKIE_SAMESITE` | `lax` | Use a stricter value only after testing login and CSRF flows. |
|
||||
| `AUTH_COOKIE_DOMAIN` | empty | Set only when the API and WebUI intentionally share a parent domain. |
|
||||
| `GOVOPLAN_HTTP_HSTS_SECONDS` | `31536000` in production, otherwise `0` | Emitted only for HTTPS requests. Set `0` while rehearsing a deployment that is not yet HTTPS-only. |
|
||||
| `GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES` | `536870912` (512 MiB) | Deployment hard ceiling; file and module APIs apply their own lower limits where appropriate. |
|
||||
|
||||
Interactive password login is enabled with fixed-window limits of 10 failures
|
||||
per normalized identity and 100 failures per direct client over 900 seconds.
|
||||
`AUTH_LOGIN_THROTTLE_*` settings change those limits. Counters use `REDIS_URL`
|
||||
when Redis is reachable so replicas share state; a bounded process-local
|
||||
fallback keeps development and Redis outages functional, with per-process
|
||||
enforcement until Redis recovers.
|
||||
|
||||
### Outbound Connector Egress
|
||||
|
||||
| Setting | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS` | `true` in dev/test, otherwise `false` | Deployment-wide decision. Set `true` only when pinned HTTP(S), DAV, SMTP, or IMAP transports must reach internal addresses. It does not enable an SDK transport that cannot pin every peer. |
|
||||
| `GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES` | `16777216` (16 MiB) | Maximum buffered JSON, XML, iCalendar, vCard, catalog, and connector error response. |
|
||||
| `GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES` | `536870912` (512 MiB) | Hard upper bound for a single remote file; module upload limits may be lower. |
|
||||
| `GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST` | empty | Comma-separated exact environment names usable by deployment-owned connector profiles. Tenant/API-managed profiles cannot select process variables, even when a name is listed. |
|
||||
| `GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST` | empty | Comma-separated exact absolute CA bundle paths. Mount the same files at the same paths on every API and connector worker. |
|
||||
|
||||
Production-like configuration validation requires the private-network choice to
|
||||
be explicit. HTTP connector downloads are streamed up to the configured bound,
|
||||
and credential-bearing DAV redirects remain confined to their configured
|
||||
origin.
|
||||
|
||||
The urllib, HTTPX/httpcore, SMTP, and IMAP transports resolve, validate, and
|
||||
connect to the same approved address record while retaining the original host
|
||||
for HTTP Host, TLS SNI, and certificate verification. Live SMB and S3 access
|
||||
fails closed in both public-only and private-network deployments: the current
|
||||
SDK transports cannot pin every initial and secondary peer or revalidate every
|
||||
SDK-managed redirect/referral. An explicit IP endpoint does not bypass this
|
||||
rule. Production deployments should still enforce the same decision at their
|
||||
worker/container egress firewall or outbound proxy as a second boundary.
|
||||
|
||||
File connector TLS verification may be disabled only in dev/test. A custom CA
|
||||
bundle must be an existing regular file whose resolved absolute path is listed
|
||||
in `GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST`. Environment-backed file connector
|
||||
credentials are supported only in deployment-owned connector JSON and require
|
||||
their exact names in `GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST`; UI/API profiles
|
||||
must use encrypted stored credentials or a scoped secret-provider reference.
|
||||
|
||||
Public URLs are currently supplied by deployment/reverse-proxy configuration and
|
||||
module settings. Do not hardcode them in core; configuration packages should ask
|
||||
for portal, WebUI, postbox, and notification URLs when they become relevant.
|
||||
Uvicorn applies `X-Forwarded-*` only from `FORWARDED_ALLOW_IPS`; keep that value
|
||||
aligned with the reverse proxy and do not expose the application server directly
|
||||
through the same trusted address range.
|
||||
|
||||
### Module Catalogs, Licenses, And Trust Roots
|
||||
|
||||
@@ -218,7 +277,7 @@ configuration, not the core runtime contract. Store them in a local ignored
|
||||
## First Deployment Flow
|
||||
|
||||
1. Create an environment file or secret set with the runtime contract above.
|
||||
2. Install the tagged core and module packages from `requirements-release.txt`.
|
||||
2. Install the tagged core and module packages from meta `requirements-release.txt`.
|
||||
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
||||
artifact from the same release tag.
|
||||
4. Run database migrations with the target `DATABASE_URL`.
|
||||
@@ -251,29 +310,29 @@ WebUI code in the editable repositories while Docker provides PostgreSQL and
|
||||
Redis:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
The helper wrapper provides explicit lifecycle commands:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
scripts/production-like-dev.sh stop
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
tools/launch/production-like-dev.sh validate-config
|
||||
tools/launch/production-like-dev.sh seed
|
||||
tools/launch/production-like-dev.sh start
|
||||
tools/launch/production-like-dev.sh stop
|
||||
tools/launch/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The launcher uses `dev/production-like/.env` when present, otherwise the checked
|
||||
in `.env.example`. It runs:
|
||||
The launcher uses `govoplan/dev/production-like/.env` when present, otherwise
|
||||
the checked in `.env.example`. It runs:
|
||||
|
||||
- PostgreSQL on `127.0.0.1:55433`
|
||||
- Redis on `127.0.0.1:56379`
|
||||
- explicit `ENABLED_MODULES`
|
||||
- explicit migrations and `--with-dev-data` bootstrap
|
||||
- API via the module-aware devserver
|
||||
- a Celery worker for `send_email,append_sent,default`
|
||||
- a Celery worker for `send_email,append_sent,notifications,calendar,default`
|
||||
- WebUI through the Vite dev server
|
||||
- durable local files under `runtime/production-like/files`
|
||||
|
||||
@@ -286,7 +345,7 @@ deployment test.
|
||||
To stop PostgreSQL and Redis when the launcher exits:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
## Module Install/Uninstall Operations
|
||||
@@ -392,7 +451,7 @@ Run the rollback drill before relying on installer automation in a new
|
||||
environment:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/module-installer-rollback-drill.py --format json
|
||||
/mnt/DATA/git/govoplan/tools/checks/module-installer-rollback-drill.py --format json
|
||||
```
|
||||
|
||||
The drill uses temporary SQLite databases and simulated package commands. It
|
||||
|
||||
@@ -1,319 +1,16 @@
|
||||
# Gitea Issues And Wiki Workflow
|
||||
|
||||
Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes.
|
||||
The shared GovOPlaN Gitea issue, label, and wiki workflow tooling moved to the
|
||||
meta repository.
|
||||
|
||||
The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state.
|
||||
|
||||
## Initial Setup
|
||||
|
||||
The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`.
|
||||
|
||||
The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan-core.git`). Override inference when needed:
|
||||
Use:
|
||||
|
||||
```bash
|
||||
export GITEA_URL=https://git.add-ideas.de
|
||||
export GITEA_OWNER=add-ideas
|
||||
export GITEA_REPO=govoplan-core
|
||||
export GITEA_TOKEN=...
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/gitea/gitea-sync-labels.py --help
|
||||
tools/gitea/gitea-sync-wiki.py --help
|
||||
```
|
||||
|
||||
The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens:
|
||||
Canonical documentation:
|
||||
|
||||
```bash
|
||||
GITEA_TOKEN=...
|
||||
# Optional if origin inference is not enough:
|
||||
GITEA_URL=https://git.add-ideas.de
|
||||
GITEA_OWNER=add-ideas
|
||||
GITEA_REPO=govoplan-core
|
||||
```
|
||||
|
||||
For a shared credentials file outside the target repository, pass `--env-file`:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
||||
```
|
||||
|
||||
Create a Gitea token with issue read/write access and label-management
|
||||
permission for the repository. On scoped-token instances, this usually means
|
||||
issue read/write and, if label writes are rejected, repository write permission
|
||||
too.
|
||||
|
||||
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
|
||||
Creating or updating organization labels requires a token with
|
||||
`write:organization`. Repository label management only needs repository label
|
||||
permission, but it duplicates the taxonomy into each repository.
|
||||
|
||||
Preview and apply labels:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py
|
||||
./scripts/gitea-sync-labels.py --apply
|
||||
```
|
||||
|
||||
Preview and apply the shared taxonomy as organization labels:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
|
||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
The import helpers resolve repository labels and organization labels. Repository
|
||||
labels win when a repository defines the same name locally, but a repository does
|
||||
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
|
||||
`module/*`, `area/*`, `source/*`, or `codex/*` label.
|
||||
|
||||
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
||||
|
||||
## Multiple Repositories And Workspaces
|
||||
|
||||
The helper scripts are path-based and can run from this core checkout against any repository with a Gitea remote:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply
|
||||
./scripts/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail
|
||||
./scripts/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress
|
||||
```
|
||||
|
||||
Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH.
|
||||
|
||||
When one core checkout drives another workspace, prefer `--env-file` for shared credentials instead of putting `GITEA_REPO` in the core `.env`; a repo-specific `GITEA_REPO` can accidentally override target inference.
|
||||
|
||||
For non-GovOPlaN projects, provide a project-specific label file and module/project label:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-labels.py \
|
||||
--root /path/to/project \
|
||||
--labels-file /path/to/project/docs/gitea-labels.json \
|
||||
--apply
|
||||
|
||||
./scripts/gitea-todo-import.py \
|
||||
--root /path/to/project \
|
||||
--module-label project/example \
|
||||
--extra-label area/backend
|
||||
```
|
||||
|
||||
If another project does not use `area/*` labels, disable area inference:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-todo-import.py \
|
||||
--root /path/to/project \
|
||||
--module-label project/example \
|
||||
--no-area-labels
|
||||
```
|
||||
|
||||
Install or refresh the shared issue templates in sibling or external repositories:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail
|
||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply
|
||||
```
|
||||
|
||||
The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-<name>` checkout maps to `module/<name>`. For another workspace or repository name, pass an explicit label:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-install-workflow.py /path/to/repo --module-label module/example --apply
|
||||
```
|
||||
|
||||
Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in core and run the sync script from core.
|
||||
|
||||
For a fully portable workflow kit, copy these files into the other project:
|
||||
|
||||
- `scripts/gitea_common.py`
|
||||
- `scripts/gitea-sync-labels.py`
|
||||
- `scripts/gitea-todo-import.py`
|
||||
- `scripts/gitea-codex-note.py`
|
||||
- `scripts/gitea-install-workflow.py`
|
||||
- `.gitea/ISSUE_TEMPLATE/*`
|
||||
- `.gitea/PULL_REQUEST_TEMPLATE.md`
|
||||
- a project-specific `docs/gitea-labels.json`
|
||||
|
||||
Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup.
|
||||
|
||||
## Label Taxonomy
|
||||
|
||||
Use one `type/*` label:
|
||||
|
||||
- `type/bug`
|
||||
- `type/feature`
|
||||
- `type/task`
|
||||
- `type/debt`
|
||||
- `type/docs`
|
||||
|
||||
Use one `status/*` label while the issue is open:
|
||||
|
||||
- `status/triage`: needs ownership, priority, or acceptance criteria.
|
||||
- `status/ready`: ready to implement.
|
||||
- `status/in-progress`: actively being worked.
|
||||
- `status/blocked`: blocked on an external dependency, credential, or decision.
|
||||
- `status/needs-info`: blocked on clarification.
|
||||
|
||||
Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`.
|
||||
|
||||
Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository.
|
||||
|
||||
Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first.
|
||||
|
||||
## Moving TODOs Into Gitea
|
||||
|
||||
Preview inline markers:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-todo-import.py
|
||||
```
|
||||
|
||||
Create missing issues after labels are synced:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-todo-import.py --apply
|
||||
```
|
||||
|
||||
The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items.
|
||||
|
||||
When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference:
|
||||
|
||||
```python
|
||||
# TODO(gitea#123): keep only if the local pointer is still useful
|
||||
```
|
||||
|
||||
Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers.
|
||||
|
||||
For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
It scans repository and product-directory files with backlog-like names, resolves
|
||||
shared labels from the organization label catalogue where available, creates only
|
||||
missing fallback repository labels when needed, imports missing open work, and
|
||||
deduplicates reruns by hidden fingerprint and normalized title.
|
||||
|
||||
## Mirroring Project Docs Into Gitea Wikis
|
||||
|
||||
Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||
```
|
||||
|
||||
Apply the wiki mirror:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
After renaming or deleting repository docs, prune previously managed wiki pages
|
||||
that no longer have a source file:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
|
||||
```
|
||||
|
||||
The default apply path uses the Gitea wiki git repository, not one REST API
|
||||
request per page. It keeps a local checkout cache below
|
||||
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
||||
pushes that commit. This is much faster and avoids REST wiki-page timeouts.
|
||||
Use `--transport api` only when the wiki git remote is unavailable.
|
||||
|
||||
Limit a sync to one repository or one generated page while working:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --apply
|
||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply
|
||||
```
|
||||
|
||||
Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository
|
||||
sync when the set of mirrored pages changes.
|
||||
|
||||
The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related.
|
||||
|
||||
Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages.
|
||||
|
||||
Use the wiki for durable context:
|
||||
|
||||
- project overviews and architecture
|
||||
- workflows, operating notes, and setup references
|
||||
- product concepts, plans, pitches, and whitepapers
|
||||
- historical context that helps interpret issues
|
||||
|
||||
Keep active state in issues:
|
||||
|
||||
- open tasks, TODOs, feature requests, and bugs
|
||||
- priority, blocking status, and acceptance criteria
|
||||
- Codex progress updates and implementation notes
|
||||
|
||||
## Codex State Updates
|
||||
|
||||
Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes:
|
||||
|
||||
- scope understood
|
||||
- files changed
|
||||
- tests or manual checks run
|
||||
- blockers or decisions needed
|
||||
- follow-up issues created
|
||||
|
||||
Preview and post a standardized note:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-codex-note.py \
|
||||
--issue 123 \
|
||||
--status progress \
|
||||
--summary "Implemented capability metadata fallback." \
|
||||
--changed src/govoplan_core/modules/registry.py \
|
||||
--test "./.venv/bin/python -m unittest tests.test_module_system"
|
||||
|
||||
./scripts/gitea-codex-note.py \
|
||||
--issue 123 \
|
||||
--status progress \
|
||||
--summary "Implemented capability metadata fallback." \
|
||||
--changed src/govoplan_core/modules/registry.py \
|
||||
--test "./.venv/bin/python -m unittest tests.test_module_system" \
|
||||
--apply
|
||||
```
|
||||
|
||||
Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded.
|
||||
|
||||
## Ownership Rules
|
||||
|
||||
Create the issue in the repository that owns the change:
|
||||
|
||||
- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components.
|
||||
- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration.
|
||||
- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations.
|
||||
- `govoplan-files`: files-specific backend, frontend, storage, and file workflows.
|
||||
- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior.
|
||||
|
||||
For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details.
|
||||
|
||||
## Cleaning Up Mirrored Sources
|
||||
|
||||
After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state.
|
||||
|
||||
Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding.
|
||||
|
||||
Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path.
|
||||
|
||||
## Docs Versus Issues
|
||||
|
||||
Keep durable facts in docs:
|
||||
|
||||
- architecture and extension points
|
||||
- command references
|
||||
- module boundaries
|
||||
- operational conventions
|
||||
|
||||
Keep changing state in Gitea:
|
||||
|
||||
- TODOs and follow-ups
|
||||
- bugs and feature requests
|
||||
- blocked status
|
||||
- acceptance criteria
|
||||
- implementation notes from active work
|
||||
|
||||
If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history.
|
||||
- `/mnt/DATA/git/govoplan/docs/GITEA_ISSUES.md`
|
||||
|
||||
@@ -1,13 +1,23 @@
|
||||
# GovOPlaN Master Roadmap
|
||||
|
||||
This roadmap is the durable product north star and sequencing guide for
|
||||
GovOPlaN as a modular platform for administrative operations. It keeps the
|
||||
product moving without turning every possible public-sector need into an
|
||||
immediate implementation track.
|
||||
This roadmap is the technical and module-sequencing companion for GovOPlaN as
|
||||
a modular platform for administrative operations. It translates the
|
||||
cross-product outcome horizons into dependency waves without turning every
|
||||
possible public-sector need into an immediate implementation track.
|
||||
|
||||
Use this document for product direction, sequencing, and module routing. Issues
|
||||
are the active backlog; this document is durable planning context and should be
|
||||
mirrored to the Gitea wiki.
|
||||
Use this document for technical sequencing, module routing, and implementation
|
||||
gates. Issues are the active backlog; this document is durable architecture
|
||||
planning context and should be mirrored to the Gitea wiki.
|
||||
|
||||
The meta repository's
|
||||
[Connected Governance Platform Roadmap](https://git.add-ideas.de/add-ideas/govoplan/src/branch/main/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md)
|
||||
describes the corresponding cross-product stakeholder visions, configurable
|
||||
service and operating configurations, connected outcome stories, and
|
||||
capability horizons. The selected five-stage delivery sequence and its gates
|
||||
are in the meta repository's
|
||||
[Reference Journey Program](https://git.add-ideas.de/add-ideas/govoplan/src/branch/main/docs/REFERENCE_JOURNEY_PROGRAM.md).
|
||||
Those product documents are canonical; this Core roadmap remains their
|
||||
technical sequencing and module-routing companion.
|
||||
|
||||
## Product Thesis
|
||||
|
||||
@@ -113,7 +123,8 @@ pattern exists.
|
||||
|
||||
## Focus Rules
|
||||
|
||||
1. Build one reference journey per wave.
|
||||
1. Build one selected reference journey stage at a time; a later capability
|
||||
cluster is not an active program merely because it appears below.
|
||||
2. Do not implement a module because the repository exists.
|
||||
3. Do not add module-to-module imports for optional behavior.
|
||||
4. Every new domain module must justify its own semantics beyond `cases`,
|
||||
@@ -138,11 +149,15 @@ pattern exists.
|
||||
| Internal work queues and tasks | `govoplan-tasks` |
|
||||
| Appointment proposals and booking | `govoplan-appointments`, `govoplan-calendar` |
|
||||
| Postbox, email, and notifications | `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications` |
|
||||
| Canonical subjects and account links | `govoplan-identity` |
|
||||
| Organizational structures, units, and functions | `govoplan-organizations` |
|
||||
| Identity-to-function assignments and directory synchronization | `govoplan-idm` |
|
||||
| Identity trust, device keys, and encrypted postbox key contracts | `govoplan-identity-trust`, `govoplan-access`, `govoplan-postbox` |
|
||||
| Service directory/catalog | `govoplan-portal` |
|
||||
| Permit/document generation | `govoplan-templates`, `govoplan-dms` |
|
||||
| Payment capture and accounting handoff | `govoplan-payments`, `govoplan-ledger` |
|
||||
| Roles, permissions, tenants, policy, audit | `govoplan-access`, `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` |
|
||||
| Authentication projection, roles, permissions, acting context | `govoplan-access` |
|
||||
| Tenants, policy, and audit | `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` |
|
||||
| External software integration | `govoplan-connectors` |
|
||||
| Recurring extraction and transformation | possible future `govoplan-datasources`, possible future `govoplan-dataflow` |
|
||||
| Reports, BI, and management visibility | `govoplan-reporting` |
|
||||
@@ -189,56 +204,82 @@ an editor applies a high-impact configuration change.
|
||||
|
||||
## Reference Journeys
|
||||
|
||||
The roadmap should be driven by three journeys.
|
||||
The active sequence is selected. Workflow remains deliberately deferred and is
|
||||
not a dependency of these journeys.
|
||||
|
||||
### Journey 1: Permit To Payment
|
||||
### Journey 1: Campaign Demonstration Composition
|
||||
|
||||
This is the primary public-administration journey.
|
||||
Campaign is the first complete proof of modular composition. Campaign owns
|
||||
intent, recipient snapshots, personalization, execution state, and delivery
|
||||
evidence. Mail owns reusable profiles, credentials, protocol policy, and
|
||||
provider execution; Campaign stores only a selected profile reference. Files
|
||||
owns storage, connector profiles, file policy, and provenance.
|
||||
|
||||
1. A person applies for a permit through the public portal.
|
||||
2. The applicant uploads required files and submits structured form data.
|
||||
3. Submission creates a case, a workflow instance, and an internal task.
|
||||
4. Completing the task creates a postbox message, a notification, and an email
|
||||
notification with an appointment proposal.
|
||||
5. The applicant accepts an appointment, which updates the calendar and the
|
||||
workflow state.
|
||||
6. During the appointment, the case is opened and the permit is generated from
|
||||
a governed template.
|
||||
7. The payment is processed and linked to the case and accounting handoff.
|
||||
8. The permit, payment evidence, communication history, audit trail, retention
|
||||
state, and records evidence remain available according to policy.
|
||||
The technical gate is a pinned Campaign/Mail/Files composition with central
|
||||
UI, adaptive user/admin/operator/integration documentation, target SMTP/IMAP
|
||||
and file-provider evidence, and explicit test/send/resend/retry/reconciliation
|
||||
semantics. Readers must not receive backend paths, worker claims, secrets, or
|
||||
raw provider diagnostics.
|
||||
|
||||
This journey proves the platform can coordinate modules without core knowing
|
||||
module internals.
|
||||
### Journey 2: Function-Bound Postbox Delivery
|
||||
|
||||
### Journey 2: Training To Certificate
|
||||
Postbox accepts delivery to an addressable postbox or a function in an
|
||||
organizational unit. Organizations owns units and functions, Identity owns
|
||||
subjects, IDM owns identity-to-function assignments and upstream sync, and
|
||||
Access resolves current roles, delegation, acting context, and permission.
|
||||
|
||||
This is the best university-administration and internal-administration journey.
|
||||
Campaign consumes a typed delivery-target capability without importing Postbox
|
||||
or identity internals. Reassignment changes future access without moving the
|
||||
message; vacancy or ambiguous acting context fails visibly; delivery, access,
|
||||
and correction remain auditable.
|
||||
|
||||
1. Course or training offer is planned.
|
||||
2. Room, trainer, resource, and capacity are booked.
|
||||
3. Participants register or are assigned.
|
||||
4. Attendance is tracked.
|
||||
5. Certificate or participation confirmation is issued.
|
||||
6. Evidence remains available through records, files, audit, and docs.
|
||||
### Journey 3: Data-Backed Templates, Reports, And Deep Launch
|
||||
|
||||
This journey keeps `booking`, `resources`, `learning`, and `certificates`
|
||||
focused instead of becoming broad ERP replacements.
|
||||
An authenticated user follows an opaque, short-lived launch reference from HIS
|
||||
or another specialist system. GovOPlaN re-authorizes the actor, resolves a
|
||||
curated data context server-side, displays source/freshness/version, and renders
|
||||
one reproducible document and report.
|
||||
|
||||
### Journey 3: Report To Resolution
|
||||
Templates owns definition/version/schema/rendering, Reporting owns source
|
||||
selection/parameters/execution/export, Files owns generated bytes, and
|
||||
connectors own protocol access. URLs do not carry credentials, arbitrary SQL,
|
||||
or trusted raw personal data. Retries are idempotent and generation evidence
|
||||
connects source, snapshot/reference, transformation, definition, parameters,
|
||||
output checksum, actor, and policy.
|
||||
|
||||
This is the internal operations and municipal issue-reporting journey.
|
||||
### Journey 4: Governed University BI Path
|
||||
|
||||
1. A person reports an issue.
|
||||
2. The issue is triaged into helpdesk, facilities, assets, or a case.
|
||||
3. Work is assigned, tracked, and escalated.
|
||||
4. Evidence, communication, and status updates are preserved.
|
||||
5. Reports show workload, SLA, recurring problems, and completion.
|
||||
Starting from the Journey 3 source contract, one bounded university dataset is
|
||||
catalogued, staged by snapshot or watermark, validated, transformed through a
|
||||
versioned lineage graph, and exposed as a policy-aware analytical data product.
|
||||
The result must preserve official-key mappings, organizational and reporting
|
||||
date semantics, quality findings, quarantine/replay, transparent calculation,
|
||||
and reproducible promotion between development, test, and production.
|
||||
|
||||
This journey prevents `helpdesk`, `issue-reporting`, `facilities`, and `assets`
|
||||
from becoming disconnected ticket silos.
|
||||
Reporting consumes the product. Create `govoplan-datasources` or
|
||||
`govoplan-dataflow` only after the concrete path proves repeated ownership that
|
||||
does not belong to connectors, Reporting, or the producing domain module.
|
||||
|
||||
## Roadmap Waves
|
||||
### Journey 5: Collaborative Document Lifecycle
|
||||
|
||||
An uploaded or generated artifact becomes a DMS document. Files continues to
|
||||
own bytes; DMS owns identity, versions, renditions, editing sessions, locks,
|
||||
comments, review, approval, comparison, and recovery; a collaboration connector
|
||||
owns provider-specific protocol behavior; Records owns later classification,
|
||||
hold, archive, and disposal.
|
||||
|
||||
The gate requires no silent lost updates, short-lived and currently authorized
|
||||
editing sessions, idempotent authenticated callbacks, visible uncertain saves,
|
||||
immutable accepted renditions, and a Records-ready handoff with stable content
|
||||
and provenance.
|
||||
|
||||
## Capability Dependency Waves
|
||||
|
||||
The waves below remain a dependency and ownership catalogue for the wider
|
||||
product vision. They are not the active delivery order. The five selected
|
||||
journeys above and the meta roadmap decide what is implemented now; other
|
||||
clusters remain dormant until a selected journey consumes them or they are
|
||||
explicitly reprioritized.
|
||||
|
||||
### Wave 0: Platform Spine
|
||||
|
||||
@@ -248,9 +289,13 @@ Refine:
|
||||
|
||||
- `govoplan-core`: module discovery, capabilities, events, migrations, release
|
||||
catalog, configuration package runtime, WebUI shell.
|
||||
- `govoplan-access`: identities, sessions, API keys, users, groups, roles,
|
||||
memberships, function assignments, delegation, RBAC decisions.
|
||||
- `govoplan-tenancy`: tenant and organizational-unit boundaries.
|
||||
- `govoplan-identity`: canonical identities and account links.
|
||||
- `govoplan-organizations`: organizational structures, units, and functions.
|
||||
- `govoplan-idm`: identity-to-function assignments, directory synchronization,
|
||||
preview, conflicts, and reconciliation.
|
||||
- `govoplan-access`: sessions, API keys, users, groups, roles, memberships,
|
||||
function-to-role projection, delegation, acting context, and RBAC decisions.
|
||||
- `govoplan-tenancy`: tenant lifecycle and tenant boundaries.
|
||||
- `govoplan-identity-trust`: initial trust contracts for device keys, public key
|
||||
directory, assurance, and later encrypted postbox key access.
|
||||
- `govoplan-policy`: policy sources, policy decisions, retention inputs.
|
||||
@@ -530,17 +575,30 @@ Before a module becomes release-included, it needs:
|
||||
- smoke test or permutation test
|
||||
- no required imports from optional modules
|
||||
|
||||
## Priority Order Summary
|
||||
## Technical Dependency Order Summary
|
||||
|
||||
1. Stabilize the platform spine.
|
||||
2. Deliver permit-to-payment MVP.
|
||||
3. Build booking and resource operations.
|
||||
4. Add learning and certificates.
|
||||
5. Add issue reporting and helpdesk.
|
||||
6. Add records, DMS, search, and transparency.
|
||||
7. Add procurement, contracts, grants, and finance handoff.
|
||||
8. Add committee and consultation workflows.
|
||||
9. Expand integration, dataflow, reporting, and operations.
|
||||
Use this active order while respecting the ownership and implementation gates
|
||||
in the capability waves:
|
||||
|
||||
1. Keep the platform/release spine green and extend connector, identity,
|
||||
external-effect, provenance, documentation, focused-view, recovery, and
|
||||
version contracts only as the current journey requires.
|
||||
2. Complete and package Campaign with Mail-owned profiles, Files, target
|
||||
delivery/recovery, central UI, and adaptive documentation.
|
||||
3. Implement function-bound Postbox delivery through
|
||||
Organizations–Identity–IDM–Access and consume it from Campaign through a
|
||||
typed capability.
|
||||
4. Implement one data-backed Templates/Reporting path and safe HIS-style deep
|
||||
launch.
|
||||
5. Extend that concrete source into one governed university analytical data
|
||||
product before generalizing data-source or dataflow ownership.
|
||||
6. Implement Files-backed DMS versions and one provider-neutral collaborative
|
||||
editing lifecycle, then connect Records handoff.
|
||||
7. Maintain already integrated Calendar/Scheduling/Poll and other foundations;
|
||||
activate another capability cluster only when the current journey needs it
|
||||
or the product roadmap explicitly reprioritizes it.
|
||||
8. Resume Workflow only by explicit product decision and constrain it with
|
||||
stable actions from one demonstrated package.
|
||||
|
||||
## Deliberate Deferrals
|
||||
|
||||
@@ -548,7 +606,8 @@ Defer these until a reference journey proves the need:
|
||||
|
||||
- full ERP replacement
|
||||
- native project management beyond connector support
|
||||
- broad BI/dataflow platform
|
||||
- an unbounded general-purpose dataflow platform; the bounded governed BI
|
||||
reference journey is selected
|
||||
- every possible public-sector protocol adapter
|
||||
- rich LMS behavior beyond training administration
|
||||
- full qualified digital signing/trust services beyond the identity-trust and
|
||||
@@ -645,19 +704,24 @@ Release composition and tag-only repository handling are documented in
|
||||
|
||||
## Next Practical Work
|
||||
|
||||
The next planning step should create or update Gitea issues for Wave 0 and Wave
|
||||
1 only. Later waves should stay as roadmap context until the permit-to-payment
|
||||
MVP is demonstrable.
|
||||
The active cross-product story is
|
||||
[`add-ideas/govoplan#14`](https://git.add-ideas.de/add-ideas/govoplan/issues/14).
|
||||
Module repositories own implementation issues; do not clone their state here.
|
||||
|
||||
Recommended immediate issue buckets:
|
||||
Immediate issue buckets:
|
||||
|
||||
- platform spine hardening
|
||||
- configuration package preflight and rollback
|
||||
- forms-runtime MVP
|
||||
- portal submission MVP
|
||||
- cases/workflow/tasks integration MVP
|
||||
- template-generated decision document
|
||||
- postbox/notification handoff
|
||||
- appointment/booking handoff
|
||||
- payment evidence handoff
|
||||
- configured documentation for the reference process
|
||||
- fail-closed connector destination pinning and private-network deployment
|
||||
control for every real transport
|
||||
- Mail-profile-only Campaign authoring/build/delivery and safe legacy failure
|
||||
- immediate audited secret deletion when a provider/profile is removed
|
||||
- Campaign central-component and role-safe UI acceptance
|
||||
- adaptive Campaign, Mail, and Files task/process/admin/operator/integration/
|
||||
security/acceptance documentation
|
||||
- target SMTP/IMAP, file-provider, queue/reconciliation, install/upgrade, and
|
||||
restore proof for the pinned Campaign reference composition
|
||||
|
||||
Once that gate is demonstrable, activate the existing Postbox model, access,
|
||||
API, inbox, and Campaign integration issues. Templates/Reporting, governed BI,
|
||||
and DMS collaboration remain durable selected direction, but should be
|
||||
decomposed only as the preceding stage stabilizes or a bounded independent
|
||||
contract can be implemented without pre-deciding target-system choices.
|
||||
|
||||
@@ -126,6 +126,15 @@ Feature modules should prefer these capabilities over direct reads of
|
||||
access/tenant ORM models when they need labels, group membership, default
|
||||
access provisioning, counts, audit actor labels, or tenant metadata.
|
||||
|
||||
Other stable runtime capabilities currently include:
|
||||
|
||||
- `identity.directory` and `identity.search`
|
||||
- `organizations.directory`
|
||||
- `idm.directory`
|
||||
- `calendar.outbox` and `calendar.scheduling`
|
||||
- `poll.scheduling`
|
||||
- `notifications.dispatch`
|
||||
|
||||
### Named Interface Contracts
|
||||
|
||||
Capabilities are runtime objects. Named interface contracts are compatibility
|
||||
@@ -147,15 +156,25 @@ intended for SemVer major-version lines. Missing optional interfaces are
|
||||
allowed, but an installed provider with an incompatible version blocks
|
||||
activation because the integration would otherwise bind to an unsafe API.
|
||||
|
||||
Current named interfaces:
|
||||
Current named interfaces, generated from the source manifests by the workspace
|
||||
contract checks, are:
|
||||
|
||||
- `files.campaign_attachments`
|
||||
- `addresses.contact_writer`, `addresses.lookup`, `addresses.recipient_source`
|
||||
- `calendar.outbox`, `calendar.scheduling`
|
||||
- `campaigns.access`, `campaigns.delivery_tasks`,
|
||||
`campaigns.mail_policy_context`, `campaigns.policy_context`,
|
||||
`campaigns.retention`
|
||||
- `dist_lists.expand`, `dist_lists.source`, `dist_lists.writer`
|
||||
- `evaluation.feedback`, `evaluation.result_aggregation`, `evaluation.scoring`
|
||||
- `files.access`, `files.campaign_attachments`
|
||||
- `mail.campaign_delivery`
|
||||
- `campaigns.access`
|
||||
- `campaigns.delivery_tasks`
|
||||
- `campaigns.mail_policy_context`
|
||||
- `campaigns.policy_context`
|
||||
- `campaigns.retention`
|
||||
- `notifications.dispatch`
|
||||
- `poll.availability_matrix`, `poll.option_selection`,
|
||||
`poll.response_collection`, `poll.signed_participation`,
|
||||
`poll.workflow_context`
|
||||
- `rest.function_publication`
|
||||
- `scheduling.candidate_slots`, `scheduling.decision_handoff`
|
||||
- `soap.operation_publication`
|
||||
|
||||
Core validates named interface contracts in three places:
|
||||
|
||||
@@ -434,6 +453,18 @@ The manifest should declare:
|
||||
- navigation metadata using serializable icon names
|
||||
- uninstall guard providers for data, migration, worker, or scheduler vetoes
|
||||
|
||||
A tenant-level managed `RoleTemplate` may set `default_authenticated=True`
|
||||
only when every authenticated tenant member must receive that narrow baseline
|
||||
while the contributing module is installed. Access derives the explicit grant
|
||||
from the active manifest set during authorization without mutating the request
|
||||
transaction. It may materialize a non-assignable role row for administration,
|
||||
but no per-user assignment is required and role edits cannot remove the
|
||||
baseline.
|
||||
This is not a shortcut for feature authorization: keep the template narrow and
|
||||
continue to enforce each domain action's own permission and resource policy.
|
||||
System-level, unmanaged, wildcard-bearing, or slug-colliding automatic
|
||||
templates are rejected by registry validation.
|
||||
|
||||
Backend nav metadata must use icon-name strings, not frontend components:
|
||||
|
||||
```python
|
||||
@@ -481,12 +512,20 @@ Rules:
|
||||
- Keep cross-module foreign-key assumptions explicit and conservative.
|
||||
- Register module metadata in `MigrationSpec` so core can discover it.
|
||||
- Optional module migrations may create multiple Alembic heads. Verification
|
||||
should compare the database heads to the configured script heads instead of
|
||||
assuming one linear revision when multiple modules are enabled.
|
||||
should compare database heads to Alembic's resolved `heads` target instead
|
||||
of assuming one linear revision when multiple modules are enabled. Owner
|
||||
heads can be dependency parents and therefore may not all appear in
|
||||
`alembic_version` after a full-graph upgrade.
|
||||
- Treat migrations as release artifacts. Unreleased migrations may be squashed
|
||||
or rewritten before a stable release; released revision IDs are immutable
|
||||
once an installation may have recorded them. Each stable release records its
|
||||
public migration heads in `docs/migration-release-baselines.json`.
|
||||
- GovOPlaN keeps two Alembic tracks. The default `release` track loads
|
||||
`versions` directories with reviewed release baselines and release-to-release
|
||||
step-up migrations. The explicit `dev` track loads `dev_versions`
|
||||
directories with the detailed development chain. Do not load both tracks for
|
||||
one migration run, and do not switch a database between tracks unless it is a
|
||||
disposable development database.
|
||||
|
||||
## Install, Uninstall, And Catalogs
|
||||
|
||||
@@ -495,7 +534,7 @@ check, maintenance-mode guard, replay state, and installer request queue.
|
||||
Package mutation is performed by `govoplan-module-installer` outside the
|
||||
FastAPI request process.
|
||||
|
||||
Official catalogs can be served as static JSON from `govoplan-web`, but core
|
||||
Official catalogs can be served as static JSON from `addideas-govoplan-website`, but core
|
||||
does not trust the website by location alone. A catalog must pass the configured
|
||||
signature, channel, freshness, and replay rules before a catalog entry can be
|
||||
planned. Catalog entries may declare `license_features`; core checks those
|
||||
@@ -699,7 +738,7 @@ Rules:
|
||||
|
||||
### Dependency Boundary Enforcement
|
||||
|
||||
The repository includes `scripts/check_dependency_boundaries.py`. It enforces the current baseline:
|
||||
The meta repository includes `tools/checks/check_dependency_boundaries.py`. It enforces the current baseline:
|
||||
|
||||
- kernel/core source may not add new direct imports of files/mail/campaign internals
|
||||
- access source may not import files/mail/campaign internals
|
||||
@@ -1208,18 +1247,18 @@ Backend verification from core:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||
/mnt/DATA/git/govoplan/tools/checks/check_dependency_boundaries.py
|
||||
```
|
||||
|
||||
`scripts/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||
`govoplan/tools/checks/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||
so developer-local npm settings do not create release-check warning noise.
|
||||
|
||||
Focused module contract and permutation verification:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-module-matrix.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-module-matrix.sh
|
||||
```
|
||||
|
||||
Core WebUI host verification:
|
||||
@@ -1233,7 +1272,7 @@ Clean generated `dist`, `.vite`, and source-tree `__pycache__` artifacts after v
|
||||
|
||||
## Release Dependency Rules
|
||||
|
||||
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. Core provides:
|
||||
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. The meta repository provides:
|
||||
|
||||
- `requirements-dev.txt` for local editable backend installs
|
||||
- `requirements-release.txt` for tagged backend module installs
|
||||
|
||||
@@ -97,6 +97,20 @@ New backend code should import policy-owned retention behavior from
|
||||
`govoplan-policy` or request the capability, not add new implementation logic
|
||||
to core.
|
||||
|
||||
The retention API DTOs live in `govoplan_core.privacy.schemas`.
|
||||
`PrivacyRetentionPolicyItem`, `PrivacyRetentionPolicyPatchItem`,
|
||||
`RETENTION_POLICY_FIELD_KEYS`, and `default_allow_lower_level_limits()` are
|
||||
platform contracts because admin, access compatibility, and policy routes expose
|
||||
the same stable retention payload shape. The policy engine's internal
|
||||
`PrivacyRetentionPolicy` and `PrivacyRetentionPolicyPatch` models stay in
|
||||
`govoplan-policy`, because they carry implementation validators and merge
|
||||
behavior that are not generic API contracts.
|
||||
|
||||
Tenant administration DTOs remain owned by `govoplan-tenancy`; access keeps
|
||||
matching compatibility DTOs only for its legacy admin surface. Admin overview
|
||||
responses remain module-local because the same counters are exposed from
|
||||
different menu contexts and are not yet a separately versioned platform API.
|
||||
|
||||
## Frontend Contract
|
||||
|
||||
Policy UIs must:
|
||||
|
||||
@@ -57,6 +57,11 @@ The trust layer should provide:
|
||||
- key rotation and epoch tracking
|
||||
- recovery policy hooks
|
||||
|
||||
Recovery must be organizationally governed. A server-held universal plaintext
|
||||
key would defeat the E2EE claim; any escrow, threshold recovery, or emergency
|
||||
grant needs an explicit assurance profile, authority/quorum, audit trail, and
|
||||
user-visible consequence.
|
||||
|
||||
## Role And Function Postboxes
|
||||
|
||||
Role-bound access needs special handling. A postbox can be bound to an
|
||||
@@ -75,6 +80,30 @@ rewrapping service:
|
||||
Key epochs are required when role membership changes. Older messages may remain
|
||||
readable according to policy, but new access must use the current epoch.
|
||||
|
||||
The function-bound container exists independently of membership. It may remain
|
||||
vacant and continue to receive ciphertext without falling back to an unrelated
|
||||
personal mailbox. Zero, one, or several incumbents are valid states. Each
|
||||
incumbent receives an independently auditable, device-bound wrapped-key path;
|
||||
the postbox is never copied into their account ownership.
|
||||
|
||||
A new assignment or hand-over rotates the function/postbox key epoch. Envelope
|
||||
encryption permits the normal rotation path to rewrap per-message data keys
|
||||
rather than rewrite large ciphertext objects; a security policy may require
|
||||
full content re-encryption for selected compromise or cryptographic-profile
|
||||
events. The history available to a new incumbent must be selected policy (all
|
||||
retained history, a bounded historical window, or assignment-time content) and
|
||||
recorded with the grant.
|
||||
|
||||
Delegation is a time-bounded represented-function grant, not a copy or
|
||||
substitution of the postbox. Expiry or withdrawal stops future key release and
|
||||
actions. It cannot revoke plaintext already decrypted, printed, exported, or
|
||||
captured outside the platform. Multiple simultaneous incumbents and delegates
|
||||
remain distinguishable in key-fetch and action evidence.
|
||||
|
||||
Postbox content and signed manifests are immutable. Correction or replacement
|
||||
creates a linked new object/version; it never silently substitutes ciphertext
|
||||
or evidence that another actor may already have inspected.
|
||||
|
||||
## External Recipients
|
||||
|
||||
External recipients may need one-time or time-limited access without a full
|
||||
|
||||
@@ -16,23 +16,23 @@ resolve modules from tagged git refs or from a package registry.
|
||||
Local development:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Release install from a core checkout plus tagged module repositories:
|
||||
Release install from the meta checkout plus tagged module repositories:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
`.[server]` is resolved relative to the current working directory. If you
|
||||
create the virtualenv elsewhere, still run the install command from the core
|
||||
checkout:
|
||||
`../govoplan-core[server]` is resolved relative to the meta requirements file.
|
||||
If you create the virtualenv elsewhere, still run the install command from the
|
||||
meta checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
@@ -40,17 +40,26 @@ cd /mnt/DATA/git/govoplan-core
|
||||
Update those refs when cutting a release:
|
||||
|
||||
```text
|
||||
govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.6
|
||||
govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.6
|
||||
govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.6
|
||||
govoplan-organizations git@git.add-ideas.de:add-ideas/govoplan-organizations.git v0.1.6
|
||||
govoplan-identity git@git.add-ideas.de:add-ideas/govoplan-identity.git v0.1.6
|
||||
govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.6
|
||||
govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.6
|
||||
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.6
|
||||
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.6
|
||||
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.6
|
||||
govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.6
|
||||
govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.8
|
||||
govoplan-organizations git@git.add-ideas.de:add-ideas/govoplan-organizations.git v0.1.8
|
||||
govoplan-identity git@git.add-ideas.de:add-ideas/govoplan-identity.git v0.1.8
|
||||
govoplan-idm git@git.add-ideas.de:add-ideas/govoplan-idm.git v0.1.8
|
||||
govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.8
|
||||
govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.8
|
||||
govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.8
|
||||
govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.8
|
||||
govoplan-dashboard git@git.add-ideas.de:add-ideas/govoplan-dashboard.git v0.1.8
|
||||
govoplan-addresses git@git.add-ideas.de:add-ideas/govoplan-addresses.git v0.1.8
|
||||
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.8
|
||||
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.8
|
||||
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.8
|
||||
govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.8
|
||||
govoplan-poll git@git.add-ideas.de:add-ideas/govoplan-poll.git v0.1.8
|
||||
govoplan-scheduling git@git.add-ideas.de:add-ideas/govoplan-scheduling.git v0.1.8
|
||||
govoplan-notifications git@git.add-ideas.de:add-ideas/govoplan-notifications.git v0.1.8
|
||||
govoplan-evaluation git@git.add-ideas.de:add-ideas/govoplan-evaluation.git v0.1.8
|
||||
govoplan-docs git@git.add-ideas.de:add-ideas/govoplan-docs.git v0.1.8
|
||||
govoplan-ops git@git.add-ideas.de:add-ideas/govoplan-ops.git v0.1.8
|
||||
```
|
||||
|
||||
## WebUI Packages
|
||||
@@ -64,24 +73,24 @@ referenced there exist, generate the committed release lockfile without
|
||||
touching the development package files:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/generate-release-lock.sh
|
||||
cd webui
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/release/generate-release-lock.sh
|
||||
cd /mnt/DATA/git/govoplan-core/webui
|
||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
||||
```
|
||||
|
||||
The module repositories include root-level npm package manifests so git
|
||||
installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`,
|
||||
`@govoplan/files-webui`, `@govoplan/mail-webui`,
|
||||
`@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository
|
||||
roots even though their source lives below `webui/src`.
|
||||
Module repositories with a frontend include root-level npm package manifests
|
||||
so the `@govoplan/*-webui` dependencies in `webui/package.release.json` can be
|
||||
resolved from repository roots even though their source lives below
|
||||
`webui/src`.
|
||||
|
||||
### Release Lockfile Strategy
|
||||
|
||||
The supported release composition currently is the full GovOPlaN product: core
|
||||
plus access, admin, tenancy, organizations, identity, policy, audit,
|
||||
dashboard, files, mail, campaign, calendar, docs, and ops. Keep one committed
|
||||
full-product release lockfile at
|
||||
The supported backend release composition is the set pinned in the meta
|
||||
repository's `requirements-release.txt`. The supported frontend composition
|
||||
is the independently buildable module set pinned in
|
||||
`webui/package.release.json`; backend-only modules do not need a frontend
|
||||
package entry. Keep one committed full-product release lockfile at
|
||||
`webui/package-lock.release.json`, generated from
|
||||
`webui/package.release.json` in a clean release workspace. Development
|
||||
`package-lock.json` may continue to point at local `file:` dependencies.
|
||||
@@ -96,18 +105,18 @@ generated in a clean release workspace from tagged git dependencies.
|
||||
|
||||
## Release Tag Script
|
||||
|
||||
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps
|
||||
The normal release path is automated by `/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh`: it bumps
|
||||
or accepts the target version, updates Python/WebUI/module manifest versions,
|
||||
commits/tags/pushes the module repositories first, regenerates
|
||||
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
||||
working tree has already been bumped, pass the current version explicitly:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/push-release-tag.sh --version 0.1.6
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/release/push-release-tag.sh --version 0.1.8
|
||||
```
|
||||
|
||||
`scripts/generate-release-catalog.py` reads installed/discovered
|
||||
`/mnt/DATA/git/govoplan/tools/release/generate-release-catalog.py` reads installed/discovered
|
||||
`ModuleManifest` objects while writing catalog entries. When a manifest is
|
||||
available, the catalog entry uses the manifest version, points package refs at
|
||||
`v<manifest.version>`, and copies `provides_interfaces` /
|
||||
@@ -126,7 +135,6 @@ are not listed in `requirements-release.txt` or `webui/package.release.json`.
|
||||
|
||||
Current tag-only module repositories:
|
||||
|
||||
- `govoplan-addresses`
|
||||
- `govoplan-appointments`
|
||||
- `govoplan-cases`
|
||||
- `govoplan-connectors`
|
||||
@@ -135,13 +143,11 @@ Current tag-only module repositories:
|
||||
- `govoplan-fit-connect`
|
||||
- `govoplan-forms`
|
||||
- `govoplan-identity-trust`
|
||||
- `govoplan-idm`
|
||||
- `govoplan-ledger`
|
||||
- `govoplan-notifications`
|
||||
- `govoplan-payments`
|
||||
- `govoplan-portal`
|
||||
- `govoplan-postbox`
|
||||
- `govoplan-reporting`
|
||||
- `govoplan-scheduling`
|
||||
- `govoplan-search`
|
||||
- `govoplan-tasks`
|
||||
- `govoplan-templates`
|
||||
@@ -157,8 +163,8 @@ running server may plan and validate package changes, but package mutation is
|
||||
performed by the separate installer daemon or an operator shell during
|
||||
maintenance mode.
|
||||
|
||||
`govoplan-web` is the public static distribution surface for official catalog
|
||||
resources:
|
||||
`addideas-govoplan-website` is the public static distribution surface for
|
||||
official catalog resources:
|
||||
|
||||
- signed module package catalogs, grouped by release channel
|
||||
- public catalog keyrings
|
||||
@@ -423,11 +429,11 @@ Dependency vulnerability checks are documented in
|
||||
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same
|
||||
The Gitea workflow in `govoplan/.gitea/workflows/dependency-audit.yml` runs the same
|
||||
check against release dependency refs on pushes, pull requests, and a weekly
|
||||
schedule.
|
||||
|
||||
@@ -587,22 +593,23 @@ entitlements without changing the source license of the repositories.
|
||||
|
||||
Production-grade distribution still needs remote registry/git artifact
|
||||
resolution before package-manager apply, a hardened catalog publishing pipeline
|
||||
in `govoplan-web`, and automated key rotation and emergency revocation drills.
|
||||
that writes to `addideas-govoplan-website`, and automated key rotation and
|
||||
emergency revocation drills.
|
||||
|
||||
## Release Catalog Publishing
|
||||
|
||||
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
||||
verified by `govoplan-core` before installer plans are accepted. Private signing
|
||||
keys must stay outside all git repositories. Public keyrings are published with
|
||||
the website.
|
||||
GovOPlaN release catalogs are published to `addideas-govoplan-website` as
|
||||
static JSON and verified by `govoplan-core` before installer plans are
|
||||
accepted. Private signing keys must stay outside all git repositories. Public
|
||||
keyrings are published with the website.
|
||||
|
||||
Create the first catalog signing key on the release machine:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
mkdir -p "$KEY_DIR"
|
||||
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
||||
./.venv/bin/python tools/release/generate-catalog-keypair.py \
|
||||
--key-id release-key-1 \
|
||||
--private-key "$KEY_DIR/release-key-1.pem" \
|
||||
--public-key "$KEY_DIR/release-key-1.pub" \
|
||||
@@ -612,12 +619,12 @@ mkdir -p "$KEY_DIR"
|
||||
Keep `release-key-1.pem` private. The generated keyring contains only public
|
||||
material.
|
||||
|
||||
Generate the signed catalog into `govoplan-web`:
|
||||
Generate the signed catalog into `addideas-govoplan-website`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/publish-release-catalog.sh \
|
||||
tools/release/publish-release-catalog.sh \
|
||||
--version <x.y.z> \
|
||||
--sequence 202607071340 \
|
||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||
@@ -626,8 +633,8 @@ scripts/publish-release-catalog.sh \
|
||||
|
||||
This writes:
|
||||
|
||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
||||
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/channels/stable.json`
|
||||
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/keyring.json`
|
||||
|
||||
The wrapper validates the catalog with core using the generated public keyring.
|
||||
|
||||
@@ -635,14 +642,14 @@ For normal module/core releases, first audit and record migration baselines,
|
||||
then tag and push the module/core repos. Finally publish the website catalog:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||
```
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/publish-release-catalog.sh \
|
||||
tools/release/publish-release-catalog.sh \
|
||||
--version <x.y.z> \
|
||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||
--build-web \
|
||||
@@ -663,16 +670,16 @@ The public keyring URL is:
|
||||
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
||||
```
|
||||
|
||||
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
||||
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` can publish the web catalog after module and core
|
||||
tags have been pushed. It runs the migration release audit in automatic mode:
|
||||
warning-only before the first recorded migration baseline, strict after a
|
||||
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
||||
mode explicitly:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/push-release-tag.sh \
|
||||
tools/release/push-release-tag.sh \
|
||||
--bump subversion \
|
||||
--strict-migration-audit \
|
||||
--publish-web-catalog \
|
||||
@@ -684,6 +691,50 @@ Use `--catalog-signing-key` more than once during a key rotation window. The
|
||||
catalog will contain multiple signatures and the public keyring will include the
|
||||
corresponding public keys.
|
||||
|
||||
## Release Doctor
|
||||
|
||||
Use `/mnt/DATA/git/govoplan/tools/release/release-doctor.py` before release preparation and again before
|
||||
publishing. It inspects repository state, local versions, release tags,
|
||||
migration audit state, release package refs, stable catalog/keyring state, and
|
||||
optionally public catalog availability. The default mode is read-only and
|
||||
offline. Status output is intentionally concise:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z>
|
||||
```
|
||||
|
||||
Use `--details` when the full findings should be printed directly:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z> --details
|
||||
```
|
||||
|
||||
For concise guidance, print only suggested next commands:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py next --target-version <x.y.z>
|
||||
```
|
||||
|
||||
For an operator-guided session, run interactive mode. The doctor asks which
|
||||
suggested command to run; mutating commands require typing `RUN` before they
|
||||
execute:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py interactive --target-version <x.y.z>
|
||||
```
|
||||
|
||||
Interactive mode starts with a compact summary and waits for an action. It can
|
||||
open the full report in the configured pager, run one suggested command, repair
|
||||
Git `safe.directory` dubious-ownership blocks after explicit confirmation, push
|
||||
all clean repositories that are ahead of their upstream, or commit and push all
|
||||
dirty repositories. The dirty-repository bulk action asks for a commit message,
|
||||
skips repositories that are behind upstream, and requires typing
|
||||
`COMMIT AND PUSH` before it stages, commits, and pushes.
|
||||
|
||||
Add `--online` when remote tag and public catalog/keyring reachability should be
|
||||
checked. Add `--json` when a CI job or another tool should consume the report.
|
||||
|
||||
On a GovOPlaN installation that should consume the official stable catalog:
|
||||
|
||||
```bash
|
||||
@@ -707,7 +758,7 @@ operator-supervised package update with restart and health checks.
|
||||
Key rotation for published catalogs:
|
||||
|
||||
1. Generate the next private key outside git.
|
||||
2. Run `publish-release-catalog.sh` with both signing keys.
|
||||
2. Run `/mnt/DATA/git/govoplan/tools/release/publish-release-catalog.sh` with both signing keys.
|
||||
3. Publish the web catalog/keyring.
|
||||
4. Roll the new public keyring into installations.
|
||||
5. Stop signing with the old key after the supported fleet trusts the new key.
|
||||
@@ -721,15 +772,15 @@ smoke check before tagging or publishing catalogs. Start the local testbed,
|
||||
then run the permutation check from the core checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
tools/checks/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
@@ -737,61 +788,110 @@ set +a
|
||||
The script checks migrations and `/health` startup for core-only, files-only,
|
||||
mail-only, campaign-only, campaign+files, campaign+mail, and full-product
|
||||
module sets. `--reset-schema` is destructive and must only be used against a
|
||||
throwaway database.
|
||||
throwaway database. Before those permutations, the required Core proof runs in
|
||||
random, test-owned schemas without modifying `public`. It exercises Files' real
|
||||
credential-owning retirement provider and proves that credential scrubbing,
|
||||
non-secret audit
|
||||
insertion, and table retirement commit together; database-injected audit and
|
||||
DDL failures roll the entire unit back. A 500 ms PostgreSQL `lock_timeout` and
|
||||
captured backend process IDs also prove that each `DROP TABLE` uses the
|
||||
installer Session connection instead of waiting through a second connection.
|
||||
The meta check enables the release-gate flag so missing PostgreSQL configuration
|
||||
or full-stack test packages are a hard failure; ordinary Core-only test discovery
|
||||
skips this integration proof. Do not pass `--skip-retirement-atomicity` when
|
||||
collecting release evidence.
|
||||
|
||||
## Migration Baselines
|
||||
|
||||
Development migrations may be small and numerous while a feature is moving.
|
||||
Before a stable release, unreleased migrations may be rewritten or squashed into
|
||||
a release-level baseline or release-to-release upgrade migration. After a
|
||||
release tag has shipped, released migration revision IDs are immutable.
|
||||
GovOPlaN keeps those detailed migrations on an explicit development track and
|
||||
publishes reviewed release shortcuts on the release track. Before a stable
|
||||
release, unreleased development migrations may be squashed into a release-level
|
||||
baseline or release-to-release upgrade migration. After a release tag has
|
||||
shipped, released migration revision IDs are immutable.
|
||||
|
||||
The release policy is:
|
||||
|
||||
- unreleased migrations may be folded before release;
|
||||
- released migrations are never rewritten or deleted;
|
||||
- unreleased development migrations live in `dev_versions` and are not deleted
|
||||
when a release shortcut is added;
|
||||
- released migrations live in `versions` and are never rewritten or deleted;
|
||||
- future release shortcuts are additive. A new release-to-release migration
|
||||
starts from the previous recorded release heads, so installations can upgrade
|
||||
through sane release steps instead of replaying every development revision;
|
||||
- each stable release records the public migration head revisions in
|
||||
`docs/migration-release-baselines.json`;
|
||||
- `heads` records the Alembic dependency-leaf graph heads for a full graph,
|
||||
while `owner_heads` records the latest migration revision per migration
|
||||
owner for subset installs and review;
|
||||
- fresh installations should apply release-level baselines/upgrades, not
|
||||
unreleased create-then-rename churn;
|
||||
- release-to-release schema changes should be folded into one reviewed
|
||||
migration per migration owner where practical.
|
||||
|
||||
The default runtime track is `release`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_MIGRATION_TRACK=release ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||
```
|
||||
|
||||
Use `dev` only for local/disposable development databases that intentionally
|
||||
need the detailed chain:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_MIGRATION_TRACK=dev ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||
```
|
||||
|
||||
Production, release checks, operator install flows, and the normal development
|
||||
launcher should stay on the `release` track unless a fresh/disposable database
|
||||
is intentionally being used to test the detailed development chain.
|
||||
|
||||
Audit the current graph during release preparation:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python scripts/release-migration-audit.py
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py
|
||||
```
|
||||
|
||||
Audit the detailed development graph separately when a squash is prepared:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --track dev
|
||||
```
|
||||
|
||||
Generate the reviewed/manual squash checklist:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --squash-plan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --squash-plan
|
||||
```
|
||||
|
||||
After the release migrations have been reviewed and the graph is final, record
|
||||
the release baseline:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --record-release <x.y.z>
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --record-release <x.y.z>
|
||||
```
|
||||
|
||||
Use strict mode to verify that the current heads are recorded:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||
```
|
||||
|
||||
`scripts/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||
non-strict while no release baseline exists, strict after the first baseline is
|
||||
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
||||
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
||||
only for emergency/manual release work.
|
||||
|
||||
Before the first stable release, fold the current development chain into the
|
||||
first public baseline and record that baseline in
|
||||
`docs/migration-release-baselines.json`. The tracking issue is
|
||||
The first public baseline is v0.1.7. It intentionally adds release-track
|
||||
shortcuts for the unreleased v0.0.0 -> v0.1.7 development chains while keeping
|
||||
the detailed chains on the `dev` track. No production installations existed
|
||||
before that baseline, so pre-v0.1.7 development revisions are not release
|
||||
upgrade targets. Future release-to-release changes must start from a recorded
|
||||
release baseline and add a new release-track step-up instead of replacing prior
|
||||
release shortcuts. The tracking issue is
|
||||
`add-ideas/govoplan-core#223`.
|
||||
|
||||
## Related Operator Documents
|
||||
@@ -807,11 +907,11 @@ first public baseline and record that baseline in
|
||||
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
||||
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
||||
calendar, and scaffold module repositories together.
|
||||
- Update `requirements-release.txt` and `webui/package.release.json` when the
|
||||
- Update meta `requirements-release.txt` and core `webui/package.release.json` when the
|
||||
release tag changes.
|
||||
- Generate the committed full-product release lockfile from
|
||||
`package.release.json` with `scripts/generate-release-lock.sh`.
|
||||
- Run `scripts/release-migration-audit.py --strict` after recording a release
|
||||
`package.release.json` with `/mnt/DATA/git/govoplan/tools/release/generate-release-lock.sh`.
|
||||
- Run `/mnt/DATA/git/govoplan/tools/release/release-migration-audit.py --strict` after recording a release
|
||||
baseline.
|
||||
- Run the PostgreSQL release check against a disposable database.
|
||||
- Publish the signed catalog through the release catalog publishing flow above.
|
||||
|
||||
15
docs/SECURITY_AUDIT.md
Normal file
15
docs/SECURITY_AUDIT.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Security Audit Toolchain
|
||||
|
||||
The shared GovOPlaN security audit toolbox moved to the meta repository.
|
||||
|
||||
Use:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||
```
|
||||
|
||||
Canonical documentation:
|
||||
|
||||
- `/mnt/DATA/git/govoplan/docs/SECURITY_AUDIT.md`
|
||||
@@ -22,7 +22,7 @@ daemon. The API server must not run package managers from request handlers.
|
||||
Generate a self-hosted template:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||
--profile self-hosted \
|
||||
--generate-secrets \
|
||||
@@ -40,40 +40,46 @@ set +a
|
||||
|
||||
The command reports all known blockers at once. Production-like/self-hosted
|
||||
profiles require explicit `APP_ENV`, `DATABASE_URL`, `MASTER_KEY_B64`,
|
||||
`ENABLED_MODULES`, and `CORS_ORIGINS`. Production rejects SQLite, development
|
||||
`ENABLED_MODULES`, `CORS_ORIGINS`, `GOVOPLAN_TRUSTED_HOSTS`, and a deployment-wide decision for
|
||||
`GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS`. Production rejects SQLite, development
|
||||
bootstrap, insecure auth cookies, and unsigned catalog trust roots when a
|
||||
catalog source is configured.
|
||||
|
||||
Connector process-secret names and custom CA files are deployment-owned through
|
||||
the exact, default-empty `GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST` and
|
||||
`GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST`; tenant/API configuration cannot widen
|
||||
either boundary.
|
||||
|
||||
## Production-Like Dev Stack
|
||||
|
||||
Use the local production-like wrapper for repeatable rehearsal:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/production-like-dev.sh validate-config
|
||||
tools/launch/production-like-dev.sh seed
|
||||
tools/launch/production-like-dev.sh start
|
||||
```
|
||||
|
||||
Stop Docker dependencies:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh stop
|
||||
tools/launch/production-like-dev.sh stop
|
||||
```
|
||||
|
||||
Reset all profile data:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
tools/launch/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The start command delegates to `scripts/launch-production-like-dev.sh`, which
|
||||
The start command delegates to `tools/launch/launch-production-like-dev.sh`, which
|
||||
runs API, worker, and WebUI in the foreground. Stop those processes with
|
||||
`Ctrl+C` in the launcher terminal.
|
||||
|
||||
## Module Boundary Gate
|
||||
|
||||
`scripts/check_dependency_boundaries.py` is part of the focused verification
|
||||
`govoplan/tools/checks/check_dependency_boundaries.py` is part of the focused verification
|
||||
path. It checks backend imports and WebUI package/source imports so modules do
|
||||
not grow hidden runtime dependencies on each other. Feature modules should
|
||||
integrate through core capabilities, backend APIs/events, route contributions,
|
||||
|
||||
21
docs/THROTTLING.md
Normal file
21
docs/THROTTLING.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# Shared fixed-window throttling
|
||||
|
||||
Core exposes `govoplan_core.core.throttling` for security-sensitive endpoints
|
||||
that need bounded fixed-window counters. Subjects are SHA-256 hashed before they
|
||||
become store keys. A configured Redis instance provides atomic counters shared
|
||||
across API workers; development, a missing Redis configuration, and temporary
|
||||
Redis outages use a bounded process-local fallback. When Redis fails, local
|
||||
attempts are still mirrored so losing the distributed store does not reset the
|
||||
active worker's protection window.
|
||||
|
||||
Callers define one or more `ThrottleDimension` values with a controlled
|
||||
namespace, a subject and a positive limit. They must call `check` before an
|
||||
expensive verifier, `record` after a failed attempt, and may `reset` the relevant
|
||||
dimension after successful verification. A blocked decision includes a
|
||||
`retry_after_seconds` value suitable for an HTTP `Retry-After` header.
|
||||
|
||||
The first consumer is Scheduling's anonymous participation password challenge.
|
||||
Its namespace is `poll-participation-password`; its subject combines tenant,
|
||||
scheduling request and Poll's non-secret invitation-token fingerprint. Access's
|
||||
login throttle predates this primitive and should be migrated onto it in a
|
||||
separate compatibility-preserving slice.
|
||||
@@ -38,6 +38,18 @@ contestability, responsibility, and traceability at the point of action.
|
||||
| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit |
|
||||
| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records |
|
||||
| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments |
|
||||
| UX-015 | Core owns the platform appearance contract. Modules must use shared CSS tokens and shared controls for theme-aware UI; they must not define independent light/dark palette systems. | Accepted | Core shell and all module WebUIs |
|
||||
| UX-016 | Full-page create/edit surfaces keep `Discard` and the named `Save …` action in the upper-right page action cluster, with Save at the far right. Their position remains stable through validation and loading states. | Accepted | All full-page create/edit surfaces |
|
||||
| UX-017 | Table row actions use icon-only controls in a stable rightmost column and intent order: inspect/open, edit, copy, transfer/share/download, retry/restore, destructive action last. Every icon requires a translated accessible name and tooltip. | Accepted | All structured tables |
|
||||
| UX-018 | A collapsible card containing only one table gives the table the card's full available body, without decorative inner wrappers, duplicate padding, max-widths, or nested scrolling. | Accepted | List, detail, workflow, and configuration surfaces |
|
||||
| UX-019 | Focused-view precedence is manual session pin, current-task suggestion, user default, role/tenant default, then the full interface. The active source and a full-interface escape remain visible; a suggested view never changes authorization or implies consent. | Accepted | Shell, modules, future workflow composition |
|
||||
| UX-020 | Centrally exported Core components are mandatory wherever their contract covers the interaction. A custom reusable control, presentation primitive, or module-local substitute requires explicit product-owner authorization, a narrowly specific purpose, and documented rationale and scope; it must not duplicate a central component. | Accepted | Core WebUI and all module WebUIs |
|
||||
| UX-021 | A collection-wide create action belongs in that collection's page heading and is not duplicated in a persistent side panel. When a side panel is the creation surface, it is present for the creation view only. | Accepted | List-detail, directory, and create surfaces |
|
||||
| UX-022 | Use central `Card` components for logical sections, `DataGrid` for tabular row collections and their ordered actions, and `ToggleSwitch` for boolean settings. Repeatable people/contact editors use one structured row per person with name, email address, and actions; free-form address parsing is reserved for an explicitly designed bulk-import flow. | Accepted | All WebUI forms and collection editors |
|
||||
| UX-023 | `FieldLabel` is the standard label/help surface for every field that is not self-explanatory. Any field rendered without it must be recorded in the omission register below, including its accessible-name source and rationale. Users may hide inline help markers through their persisted interface preference; the field label itself remains visible. | Accepted | All Core and module forms |
|
||||
| UX-024 | Explicit `Discard` actions and dirty in-application navigation use the shared `UnsavedChangesProvider` dialog. A page registers save/discard behavior with `useUnsavedDraftGuard`; its Discard button calls `requestDiscard`, and route changes use `useGuardedNavigate` or `requestNavigation`. | Accepted | All create/edit surfaces |
|
||||
| UX-025 | `window.alert` and the global `alert` function are prohibited. A narrowly necessary exception requires product-owner authorization and an entry in the alert exception register before implementation. | Accepted | All WebUI code |
|
||||
| UX-026 | A table defines one stable ordered action set. A row-level unavailable action remains in its normal position and is disabled, preferably with `disabledReason`; structurally irrelevant actions are omitted for the entire table. Empty rows reserve the same slots so their Add action stays in the normal left-most action position. | Accepted | All structured tables |
|
||||
|
||||
## Confirmed Implementation Decisions
|
||||
|
||||
@@ -138,6 +150,121 @@ adaptive form, not force a linear wizard.
|
||||
- Wizard shells remain available for assisted setup, first-run guidance,
|
||||
imports, discovery-heavy flows, and operational preflight workflows.
|
||||
|
||||
### DUE-008: Platform Theme Contract
|
||||
|
||||
Decision: the WebUI shell exposes a small, stable appearance contract based on
|
||||
shared CSS tokens and persisted user preference selection.
|
||||
|
||||
- Core applies `system`, `light`, and `dark` preferences at the document root.
|
||||
- Core owns shared tokens such as `--bg`, `--bar`, `--panel`, `--surface`,
|
||||
`--line`, `--line-dark`, `--text`, `--text-strong`, `--muted`, semantic
|
||||
status colors, radii, shadows, and disabled-control colors.
|
||||
- Modules must style new UI with these tokens and shared controls. Module-local
|
||||
CSS may tune layout and spacing, but it must not introduce a separate
|
||||
appearance system.
|
||||
- Appearance controls live in user settings first. Tenant defaults and policy
|
||||
enforcement can be added later without changing the token contract.
|
||||
- Visual preview in settings is illustrative; it must reflect token families,
|
||||
not become a second theme implementation.
|
||||
|
||||
### DUE-009: Central Component And Exception Contract
|
||||
|
||||
Decision: module interfaces are compositions of the components exported by
|
||||
`@govoplan/core-webui`. When Core already owns the matching interaction, using
|
||||
the central component is required rather than preferred.
|
||||
|
||||
A route or domain-specific page composed from central components is ordinary
|
||||
module composition. A new reusable UI control, presentation primitive, or
|
||||
module-local substitute is a custom component. Before one is implemented, the
|
||||
product owner must explicitly authorize it and the owning decision or issue must
|
||||
record:
|
||||
|
||||
- its single, narrowly defined purpose and intended consumers
|
||||
- why central components or their composition cannot meet that purpose
|
||||
- the permitted scope and the boundary it must not grow beyond
|
||||
- accessibility, reachable states, theme behavior, and test expectations
|
||||
- whether the component remains domain-owned or is a candidate for Core
|
||||
|
||||
Custom components must not duplicate, fork, or cosmetically replace a central
|
||||
component. An existing local implementation does not grant an exception. If a
|
||||
central contract later covers the need, migrate to it unless the product owner
|
||||
explicitly retains the exception.
|
||||
|
||||
### DUE-010: Scheduling Request Reference Composition
|
||||
|
||||
Decision: Scheduling requests provide a concrete reference application of the
|
||||
universal placement and component rules.
|
||||
|
||||
- The persistent left panel stacks `My scheduling requests` and `Scheduling
|
||||
requests for me`; it is list context, not a second creation affordance.
|
||||
- The left panel's `Scheduling requests` header owns one `Add` action. It opens
|
||||
the shared view/create/edit surface in the right main panel.
|
||||
- Basic information, Calendar integration, candidate slots, and participants
|
||||
use the central `Card` component as four logical sections.
|
||||
- Candidate slots and participants use the central `DataGrid`, including its
|
||||
standard row-action placement and order.
|
||||
- Calendar integration uses the central `ToggleSwitch`, with its dependent
|
||||
controls shown when enabled.
|
||||
- Each participant is edited as one structured row with name, email address,
|
||||
and actions. The normal editor does not parse a free-form list of addresses;
|
||||
that interaction requires a separate, explicitly designed bulk-import flow.
|
||||
|
||||
Equivalent list/create/edit surfaces use the same underlying rules. These are
|
||||
not Scheduling-local component variants.
|
||||
|
||||
### DUE-011: Field Help, Discard, And Table Action Contracts
|
||||
|
||||
Decision: the central components own these interactions; modules compose them
|
||||
instead of reproducing their behavior.
|
||||
|
||||
- `FormField` and `ToggleSwitch` already render `FieldLabel`. Direct field
|
||||
compositions use `FieldLabel` explicitly when the meaning or limitation is
|
||||
not self-explanatory.
|
||||
- `help` content is contextual guidance, not the accessible name. The persisted
|
||||
`show_inline_help_hints` user preference hides only the `InlineHelp` marker by
|
||||
applying `ui-hide-help-hints` at the document root.
|
||||
- A dirty editor registers once with `useUnsavedDraftGuard`. An explicit
|
||||
Discard button calls `useUnsavedChanges().requestDiscard(afterResolve)`; SPA
|
||||
navigation uses `useGuardedNavigate` or `requestNavigation`. Both paths show
|
||||
the same shared unsaved-changes dialog. A browser tab/window unload remains a
|
||||
browser-controlled confirmation because browsers do not permit a custom
|
||||
modal at that boundary.
|
||||
- `TableActionGroup` receives the table's stable action set. Use `disabled` and
|
||||
`disabledReason` for row state; omit an action only when that action does not
|
||||
belong to the table. `minimumSlots` reserves trailing positions for an empty
|
||||
row. `DataGridEmptyAction` does this for the standard add/move/remove layout.
|
||||
- A paginated `DataGrid` has exactly one query owner. Client mode receives the
|
||||
complete logical row set and applies filtering and sorting before slicing a
|
||||
page. Server mode receives only the loaded page, requires `onQueryChange`,
|
||||
and the backend applies every emitted filter/sort before pagination while
|
||||
returning `totalRows` for the filtered result. Server list filters declare
|
||||
their complete option domain instead of deriving it from the loaded page.
|
||||
External filter affordances such as summary-count shortcuts update the
|
||||
grid's `query` contract; the grid header controls and backend query therefore
|
||||
always display and execute the same filter state.
|
||||
- Feedback and confirmation use `Dialog`, `ConfirmDialog`, or
|
||||
`DismissibleAlert`. They never fall back to `window.alert`.
|
||||
|
||||
#### FieldLabel Omission Register
|
||||
|
||||
Every Core field surface that intentionally does not render `FieldLabel` is
|
||||
listed here. Module repositories keep an equivalent register in their durable
|
||||
UI documentation until a central cross-repository audit is available.
|
||||
|
||||
| Core scope | Why `FieldLabel` is omitted | Accessible/context label source |
|
||||
| --- | --- | --- |
|
||||
| `PasswordField`, `ColorPickerField`, `DateField`, `TimeField`, and `DateTimeField` input internals | These are label-neutral composite primitives and are placed inside `FormField`/`FieldLabel` by the consuming form. Rendering another label inside the primitive would duplicate it. | Enclosing label; a direct consumer must pass an accessible name and record that direct composition here. |
|
||||
| `ToggleSwitch` native checkbox | The shared component already renders its visible text through `FieldLabel`; the native input must not render a second label. | The enclosing native label and derived `aria-label`. |
|
||||
| `FileDropZone` hidden file input | The input is an implementation detail of the labelled keyboard-operable drop target. | Drop target text and `inputLabel`/`aria-label`. |
|
||||
| `AdminSelectionList` and `DataGrid` list-filter checkboxes | Each option is self-explanatory and already enclosed by its visible option label. | Enclosing native option label. |
|
||||
| `EmailAddressInput` compact Name and Email fields | These two conventional fields are self-explanatory in the compact address popover; richer address guidance belongs to the enclosing field. | Visible native labels; the free-form editor also has a descriptive `aria-label`. |
|
||||
| `DataGrid` page-size, filter, and inline cell editors | The surrounding column header/filter heading supplies field context; repeating a labelled help marker in every cell would add noise. | Column header, filter heading/native label, or generated cell `aria-label`. |
|
||||
| Retention-policy value controls | `PolicyRow` owns the field label, help, effective value, and provenance for its control. | The containing `PolicyRow` label/help contract. |
|
||||
|
||||
#### Alert Exception Register
|
||||
|
||||
No `window.alert` or global `alert` exception is authorized.
|
||||
|
||||
## Implementation Sequence
|
||||
|
||||
| Phase | Scope | Output |
|
||||
@@ -200,6 +327,22 @@ Every new or changed admin/configuration surface should answer:
|
||||
- Does the action surface show consequence, reversibility, and audit evidence
|
||||
when rights, duties, records, money, communication, external systems, or
|
||||
workflow state are affected?
|
||||
- Does the surface use every applicable central Core component? If it contains
|
||||
a custom component, is the product-owner authorization, narrow purpose,
|
||||
rationale, scope, and non-duplication evidence recorded?
|
||||
- Is a collection-wide create action in the collection heading rather than
|
||||
duplicated in a persistent side panel?
|
||||
- Are logical sections, tabular collections, boolean settings, and repeatable
|
||||
people/contact rows composed with `Card`, `DataGrid`, `ToggleSwitch`, and one
|
||||
structured row per person respectively?
|
||||
- Does every non-self-explanatory field use `FieldLabel`, and is every omission
|
||||
recorded with its rationale and accessible-name source?
|
||||
- Do explicit Discard and dirty navigation use the shared unsaved-changes
|
||||
registration/dialog rather than a page-local confirmation?
|
||||
- Does every row retain the table's action set in the same order, disabling
|
||||
unavailable actions and reserving the same empty-row slots?
|
||||
- Is feedback rendered with a central dialog/alert component, with no
|
||||
unauthorized `window.alert` or global `alert` call?
|
||||
- If automation is involved, can the user see the trigger, system actor,
|
||||
observed effects, and failure/manual-intervention state?
|
||||
- Are technical details available without being the first thing the user sees?
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
Commands:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
Status: remediated.
|
||||
@@ -45,15 +45,15 @@ Remediation applied on 2026-07-09:
|
||||
- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to
|
||||
`pyzipper==0.4.0`
|
||||
- upgraded the local audit environment to `pip==26.1.2`
|
||||
- removed the obsolete local `govoplan-module-multimailer` editable install
|
||||
- removed the obsolete local mailer-module editable install
|
||||
from the audit environment so the audit reflects the split module product
|
||||
|
||||
Post-remediation result:
|
||||
|
||||
- `bash scripts/check-dependency-audits.sh`: passed, no known Python
|
||||
- `bash tools/checks/check-dependency-audits.sh`: passed, no known Python
|
||||
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
||||
- `python -m pip check`: passed.
|
||||
- `bash scripts/check-module-matrix.sh`: passed.
|
||||
- `bash tools/checks/check-module-matrix.sh`: passed.
|
||||
- `python -m unittest tests.test_api_smoke`: passed.
|
||||
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
||||
|
||||
|
||||
@@ -1,416 +0,0 @@
|
||||
[
|
||||
{
|
||||
"name": "type/bug",
|
||||
"color": "d73a4a",
|
||||
"description": "A reproducible defect, regression, or incorrect behavior.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/feature",
|
||||
"color": "0e8a16",
|
||||
"description": "New user-visible behavior or platform capability.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/task",
|
||||
"color": "1d76db",
|
||||
"description": "Implementation, maintenance, migration, or operational work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/debt",
|
||||
"color": "fbca04",
|
||||
"description": "Cleanup, refactoring, risk reduction, or deferred engineering work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/docs",
|
||||
"color": "5319e7",
|
||||
"description": "Documentation, process, or developer workflow work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p0",
|
||||
"color": "b60205",
|
||||
"description": "Immediate stop-the-line priority.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p1",
|
||||
"color": "d93f0b",
|
||||
"description": "High priority for the next focused work window.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p2",
|
||||
"color": "fbca04",
|
||||
"description": "Normal planned priority.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p3",
|
||||
"color": "c2e0c6",
|
||||
"description": "Low priority or opportunistic cleanup.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/triage",
|
||||
"color": "d4c5f9",
|
||||
"description": "Needs review, ownership, priority, or acceptance criteria.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/ready",
|
||||
"color": "0e8a16",
|
||||
"description": "Ready for implementation.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/in-progress",
|
||||
"color": "0052cc",
|
||||
"description": "Currently being worked.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/blocked",
|
||||
"color": "b60205",
|
||||
"description": "Cannot progress without a decision, dependency, credential, or external change.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/needs-info",
|
||||
"color": "f9d0c4",
|
||||
"description": "Needs clarifying input before implementation can proceed safely.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "module/access",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/addresses",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Addresses module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/admin",
|
||||
"color": "006b75",
|
||||
"description": "GovOPlaN Admin module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/appointments",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN Appointments module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/audit",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN Audit module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/calendar",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Calendar module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/campaign",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN campaign module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/cases",
|
||||
"color": "1d76db",
|
||||
"description": "GovOPlaN Cases module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/connectors",
|
||||
"color": "c2e0c6",
|
||||
"description": "GovOPlaN Connectors module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/core",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN core runner, shared primitives, shell, or extension points.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/dms",
|
||||
"color": "c5def5",
|
||||
"description": "GovOPlaN Dms module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/erp",
|
||||
"color": "fef2c0",
|
||||
"description": "GovOPlaN Erp module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/files",
|
||||
"color": "006b75",
|
||||
"description": "GovOPlaN files module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/fit-connect",
|
||||
"color": "fbca04",
|
||||
"description": "GovOPlaN Fit Connect module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/forms",
|
||||
"color": "f9d0c4",
|
||||
"description": "GovOPlaN Forms module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/identity-trust",
|
||||
"color": "d4c5f9",
|
||||
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/identity",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Identity module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/idm",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN Idm module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/ledger",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Ledger module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/mail",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN mail module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/notifications",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN Notifications module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/ops",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN Ops module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/organizations",
|
||||
"color": "bfdadc",
|
||||
"description": "GovOPlaN Organizations module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/payments",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Payments module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/policy",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Policy module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/portal",
|
||||
"color": "1d76db",
|
||||
"description": "GovOPlaN Portal module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/postbox",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Postbox module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/reporting",
|
||||
"color": "c2e0c6",
|
||||
"description": "GovOPlaN Reporting module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/search",
|
||||
"color": "bfdadc",
|
||||
"description": "GovOPlaN Search module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/scheduling",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Scheduling module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/tasks",
|
||||
"color": "c5def5",
|
||||
"description": "GovOPlaN Tasks module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/templates",
|
||||
"color": "fef2c0",
|
||||
"description": "GovOPlaN Templates module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/tenancy",
|
||||
"color": "e4e669",
|
||||
"description": "GovOPlaN Tenancy module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/web",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN public website, product page, or publication content.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/workflow",
|
||||
"color": "f9d0c4",
|
||||
"description": "GovOPlaN Workflow module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xoev",
|
||||
"color": "d4c5f9",
|
||||
"description": "GovOPlaN Xoev module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xrechnung",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN Xrechnung module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xta-osci",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Xta Osci module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/auth",
|
||||
"color": "bfd4f2",
|
||||
"description": "Authentication, sessions, access bootstrap, or login behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/tenancy",
|
||||
"color": "bfdadc",
|
||||
"description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/rbac",
|
||||
"color": "c5def5",
|
||||
"description": "Permissions, roles, delegation, or authorization policy.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/governance",
|
||||
"color": "fef2c0",
|
||||
"description": "Governance policy, audit, privacy, retention, or compliance behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/module-system",
|
||||
"color": "bfe5bf",
|
||||
"description": "Module discovery, manifests, capabilities, routing, or optional integrations.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/migrations",
|
||||
"color": "e4e669",
|
||||
"description": "Alembic migrations, schema bootstrap, or persistence evolution.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/webui",
|
||||
"color": "1d76db",
|
||||
"description": "Shared WebUI shell, frontend components, routing, or frontend tests.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/api",
|
||||
"color": "5319e7",
|
||||
"description": "HTTP API contracts, routers, schemas, or API smoke behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/db",
|
||||
"color": "0e8a16",
|
||||
"description": "Database sessions, models, transactions, or persistence primitives.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/devex",
|
||||
"color": "bfd4f2",
|
||||
"description": "Local developer workflow, scripts, tests, tooling, or release helpers.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/release",
|
||||
"color": "c2e0c6",
|
||||
"description": "Versioning, release locks, tags, packaging, or dependency pins.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/docs",
|
||||
"color": "5319e7",
|
||||
"description": "Durable documentation and project guidance.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/marketing",
|
||||
"color": "f9d0c4",
|
||||
"description": "Public website, product messaging, publication copy, or legal page content.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "source/todo-scan",
|
||||
"color": "cccccc",
|
||||
"description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "source/backlog-import",
|
||||
"color": "bfdadc",
|
||||
"description": "Imported from markdown backlog, roadmap, plan, or TODO files.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "codex/ready",
|
||||
"color": "0e8a16",
|
||||
"description": "Suitable for Codex to pick up with the existing issue context.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "codex/needs-human",
|
||||
"color": "f9d0c4",
|
||||
"description": "Needs an explicit human decision before Codex should implement.",
|
||||
"exclusive": false
|
||||
}
|
||||
]
|
||||
@@ -10,18 +10,6 @@
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "3d4e5f708192"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revision": "4f2a9c8e7b6d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revision": "5c6d7e8f9a10"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revision": "6d7e8f9a0b1c"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
@@ -91,9 +79,590 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-11T00:19:07Z",
|
||||
"recorded_at": "2026-07-11T01:39:45Z",
|
||||
"release": "0.1.7",
|
||||
"squash_policy": "reviewed-manual"
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
},
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "2c3d4e5f7081"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "3d4e5f708192"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "9e0f1a2b3c4d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"9e0f1a2b3c4d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"2c3d4e5f7081"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"3d4e5f708192"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-20T02:45:41Z",
|
||||
"release": "0.1.8",
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
},
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "4d5e6f7a9203"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "608192abcdef"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revision": "6e7f8a9b0c1d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revision": "6f7a8b9c0d1e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "af1b2c3d4e5f"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revision": "c9d4e7f1a2b3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revision": "e1f2a4b5c6d"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revisions": [
|
||||
"e1f2a4b5c6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"af1b2c3d4e5f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"4d5e6f7a9203"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"608192abcdef"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revisions": [
|
||||
"6f7a8b9c0d1e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revisions": [
|
||||
"6e7f8a9b0c1d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revisions": [
|
||||
"c9d4e7f1a2b3"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-22T02:42:27Z",
|
||||
"release": "0.1.11",
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
},
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "608192abcdef"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revision": "6e7f8a9b0c1d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revision": "6f7a8b9c0d1e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "af1b2c3d4e5f"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revision": "c9d4e7f1a2b3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "d8b3e2c1f4a5"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revision": "e1f2a4b5c6d"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revisions": [
|
||||
"e1f2a4b5c6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"af1b2c3d4e5f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"d8b3e2c1f4a5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"608192abcdef"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revisions": [
|
||||
"6f7a8b9c0d1e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revisions": [
|
||||
"6e7f8a9b0c1d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revisions": [
|
||||
"c9d4e7f1a2b3"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-22T07:06:21Z",
|
||||
"release": "0.1.12",
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
},
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "608192abcdef"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revision": "6e7f8a9b0c1d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revision": "6f7a8b9c0d1e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "af1b2c3d4e5f"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revision": "c9d4e7f1a2b3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "d8b3e2c1f4a5"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revision": "e1f2a4b5c6d"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revisions": [
|
||||
"e1f2a4b5c6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"af1b2c3d4e5f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"d8b3e2c1f4a5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"608192abcdef"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revisions": [
|
||||
"6f7a8b9c0d1e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revisions": [
|
||||
"6e7f8a9b0c1d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revisions": [
|
||||
"c9d4e7f1a2b3"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-22T08:39:02Z",
|
||||
"release": "0.1.13",
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
},
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "608192abcdef"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revision": "6e7f8a9b0c1d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revision": "6f7a8b9c0d1e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "af1b2c3d4e5f"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revision": "c9d4e7f1a2b3"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "d8b3e2c1f4a5"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revision": "e1f2a4b5c6d"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-addresses",
|
||||
"revisions": [
|
||||
"e1f2a4b5c6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"af1b2c3d4e5f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"d8b3e2c1f4a5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"608192abcdef"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-notifications",
|
||||
"revisions": [
|
||||
"6f7a8b9c0d1e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-poll",
|
||||
"revisions": [
|
||||
"6e7f8a9b0c1d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-scheduling",
|
||||
"revisions": [
|
||||
"c9d4e7f1a2b3"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-22T18:15:01Z",
|
||||
"release": "0.1.14",
|
||||
"squash_policy": "reviewed-manual",
|
||||
"track": "release"
|
||||
}
|
||||
],
|
||||
"version": 1
|
||||
|
||||
@@ -84,7 +84,15 @@
|
||||
"provides_interfaces": [
|
||||
{
|
||||
"name": "mail.campaign_delivery",
|
||||
"version": "1.4.0"
|
||||
"version": "0.2.0"
|
||||
}
|
||||
],
|
||||
"requires_interfaces": [
|
||||
{
|
||||
"name": "campaigns.access",
|
||||
"version_min": "0.1.0",
|
||||
"version_max_exclusive": "0.2.0",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"artifact_integrity": {
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "govoplan-core"
|
||||
version = "0.1.7"
|
||||
version = "0.1.14"
|
||||
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -27,6 +27,12 @@ where = ["src"]
|
||||
[tool.setuptools.package-data]
|
||||
govoplan_core = ["py.typed"]
|
||||
|
||||
[tool.setuptools.data-files]
|
||||
"govoplan_core_runtime" = ["alembic.ini"]
|
||||
"govoplan_core_runtime/alembic" = ["alembic/env.py", "alembic/script.py.mako"]
|
||||
"govoplan_core_runtime/alembic/versions" = ["alembic/versions/*.py"]
|
||||
"govoplan_core_runtime/alembic/dev_versions" = ["alembic/dev_versions/*.py"]
|
||||
|
||||
[project.scripts]
|
||||
govoplan-config = "govoplan_core.commands.config:main"
|
||||
govoplan-devserver = "govoplan_core.devserver:main"
|
||||
@@ -42,3 +48,7 @@ dev = [
|
||||
"httpx==0.28.1",
|
||||
"httpx2>=2.5,<3",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/test_api_smoke.py" = ["E402"]
|
||||
"tests/test_module_system.py" = ["E402"]
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
-e .[server]
|
||||
-e ../govoplan-tenancy
|
||||
-e ../govoplan-organizations
|
||||
-e ../govoplan-identity
|
||||
-e ../govoplan-access
|
||||
-e ../govoplan-admin
|
||||
-e ../govoplan-policy
|
||||
-e ../govoplan-audit
|
||||
-e ../govoplan-dashboard
|
||||
-e ../govoplan-files
|
||||
-e ../govoplan-mail
|
||||
-e ../govoplan-campaign
|
||||
-e ../govoplan-calendar
|
||||
-e ../govoplan-docs
|
||||
-e ../govoplan-ops
|
||||
httpx==0.28.1
|
||||
httpx2>=2.5,<3
|
||||
pip-audit>=2.9,<3
|
||||
@@ -1,18 +0,0 @@
|
||||
# Release install from tagged module repositories.
|
||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
||||
.[server]
|
||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
|
||||
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
|
||||
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
|
||||
govoplan-idm @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-idm.git@v0.1.6
|
||||
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
|
||||
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
|
||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
|
||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
|
||||
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
|
||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
|
||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
|
||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
|
||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
|
||||
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
|
||||
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
||||
|
||||
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
||||
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
||||
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
python_status=0
|
||||
npm_status=0
|
||||
|
||||
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
||||
|
||||
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
||||
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
if importlib.util.find_spec("pip") is None:
|
||||
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
||||
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
||||
print(" python -m venv .venv", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
||||
raise SystemExit(127)
|
||||
PY
|
||||
|
||||
"$PYTHON" -m pip check
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
prefix = pathlib.Path(sys.prefix)
|
||||
legacy_patterns = (
|
||||
"__editable__.govoplan_module_multimailer-*.pth",
|
||||
"__editable___govoplan_module_multimailer_*_finder.py",
|
||||
"govoplan_module_multimailer-*.dist-info",
|
||||
)
|
||||
problems: list[pathlib.Path] = []
|
||||
|
||||
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
||||
for pattern in legacy_patterns:
|
||||
problems.extend(site_packages.glob(pattern))
|
||||
|
||||
for dist in importlib.metadata.distributions():
|
||||
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
||||
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
||||
|
||||
if problems:
|
||||
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
||||
for path in sorted(set(problems)):
|
||||
print(f" {path}", file=sys.stderr)
|
||||
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("No stale legacy multimailer package metadata found.")
|
||||
PY
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path.cwd()
|
||||
scan_roots = [
|
||||
root / "src",
|
||||
root / "tests",
|
||||
root.parent / "govoplan-access" / "src",
|
||||
root.parent / "govoplan-admin" / "src",
|
||||
root.parent / "govoplan-audit" / "src",
|
||||
root.parent / "govoplan-calendar" / "src",
|
||||
root.parent / "govoplan-campaign" / "src",
|
||||
root.parent / "govoplan-dashboard" / "src",
|
||||
root.parent / "govoplan-files" / "src",
|
||||
root.parent / "govoplan-identity" / "src",
|
||||
root.parent / "govoplan-mail" / "src",
|
||||
root.parent / "govoplan-ops" / "src",
|
||||
root.parent / "govoplan-organizations" / "src",
|
||||
root.parent / "govoplan-policy" / "src",
|
||||
root.parent / "govoplan-tenancy" / "src",
|
||||
]
|
||||
deprecated_tokens = {
|
||||
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
||||
}
|
||||
|
||||
hits: list[str] = []
|
||||
for scan_root in scan_roots:
|
||||
if not scan_root.exists():
|
||||
continue
|
||||
for path in scan_root.rglob("*.py"):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
for token, replacement in deprecated_tokens.items():
|
||||
if token in text:
|
||||
hits.append(f"{path}: deprecated {token}. {replacement}")
|
||||
|
||||
if hits:
|
||||
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
||||
print("\n".join(hits), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("Deprecated framework constant scan passed.")
|
||||
PY
|
||||
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
||||
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
||||
0|false|False|no|No)
|
||||
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
||||
;;
|
||||
auto)
|
||||
if "$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
||||
PY
|
||||
then
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
else
|
||||
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
||||
"$PYTHON" - <<'PY'
|
||||
import warnings
|
||||
|
||||
from starlette.exceptions import StarletteDeprecationWarning
|
||||
|
||||
warnings.simplefilter("error", StarletteDeprecationWarning)
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/dependency-hygiene-ping")
|
||||
def ping() -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/dependency-hygiene-ping")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"ok": True}
|
||||
|
||||
print("TestClient deprecation smoke passed.")
|
||||
PY
|
||||
fi
|
||||
|
||||
echo "Dependency hygiene check passed."
|
||||
@@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="/mnt/DATA/git/govoplan-core"
|
||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
||||
NPM="$NODE/npm"
|
||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||
|
||||
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
||||
|
||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
||||
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
||||
unset npm_config_tmp NPM_CONFIG_TMP
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||
|
||||
./.venv/bin/python - <<'PY'
|
||||
import ast
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
roots = [
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/tests"),
|
||||
]
|
||||
|
||||
errors = []
|
||||
count = 0
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
count += 1
|
||||
try:
|
||||
ast.parse(path.read_text(), filename=str(path))
|
||||
except SyntaxError as exc:
|
||||
errors.append(f"{path}:{exc.lineno}:{exc.offset}: {exc.msg}")
|
||||
|
||||
if errors:
|
||||
print("\n".join(errors))
|
||||
sys.exit(1)
|
||||
|
||||
print(f"AST syntax check passed for {count} Python files")
|
||||
PY
|
||||
|
||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
|
||||
cd "$ROOT/webui"
|
||||
"$NPM" run test:mail-components
|
||||
"$NPM" run test:module-capabilities
|
||||
"$NPM" run test:module-permutations
|
||||
|
||||
cd /mnt/DATA/git/govoplan-mail/webui
|
||||
"$NPM" run test:mail-ui
|
||||
|
||||
cd /mnt/DATA/git/govoplan-campaign/webui
|
||||
"$NPM" run test:policy-ui
|
||||
"$NPM" run test:template-preview
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
||||
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
||||
@@ -1,278 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-integration.XXXXXXXX")"
|
||||
|
||||
trap 'rm -rf "$WORK_ROOT"; rm -f "$NPM_USERCONFIG"' EXIT
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
RELEASE_VERSION="$("$PYTHON" - <<'PY'
|
||||
from pathlib import Path
|
||||
import tomllib
|
||||
|
||||
with (Path.cwd() / "pyproject.toml").open("rb") as handle:
|
||||
print(tomllib.load(handle)["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
RELEASE_TAG="${GOVOPLAN_RELEASE_TAG:-v$RELEASE_VERSION}"
|
||||
|
||||
export RELEASE_TAG RELEASE_VERSION
|
||||
export GOVOPLAN_CORE_SOURCE_ROOT="$ROOT"
|
||||
export PATH="$WEBUI_BIN:$NODE_BIN:$PATH"
|
||||
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||
unset npm_config_tmp NPM_CONFIG_TMP
|
||||
|
||||
run_step() {
|
||||
echo
|
||||
echo "==> $*"
|
||||
}
|
||||
|
||||
retry() {
|
||||
local attempt status
|
||||
for attempt in 1 2 3; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
status=$?
|
||||
if [[ "$attempt" == 3 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
sleep $((attempt * 10))
|
||||
done
|
||||
}
|
||||
|
||||
run_discovered_tests() {
|
||||
local suite_dir="$1"
|
||||
local status
|
||||
if ! find "$suite_dir" -type f -name 'test*.py' -print -quit | grep -q .; then
|
||||
echo "No unittest test files found under $suite_dir; skipping."
|
||||
return 0
|
||||
fi
|
||||
set +e
|
||||
"$PYTHON" -m unittest discover -s "$suite_dir"
|
||||
status=$?
|
||||
set -e
|
||||
if [[ "$status" == 5 ]]; then
|
||||
echo "No unittest tests discovered under $suite_dir; skipping."
|
||||
return 0
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
|
||||
install_cloned_webui_dependencies() {
|
||||
local webui_dir="$1"
|
||||
if [[ ! -f "$webui_dir/package.json" ]]; then
|
||||
echo "No package.json found under $webui_dir; skipping WebUI dependency install."
|
||||
return 0
|
||||
fi
|
||||
(
|
||||
cd "$webui_dir"
|
||||
retry "$NPM" install --prefer-online --include=dev --no-save
|
||||
)
|
||||
}
|
||||
|
||||
run_step "Validate release refs and installed package metadata"
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata as metadata
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path.cwd()
|
||||
release_requirements = root / "requirements-release.txt"
|
||||
release_webui = root / "webui" / "package.release.json"
|
||||
expected_version = os.environ["RELEASE_VERSION"]
|
||||
expected_tag = os.environ["RELEASE_TAG"]
|
||||
errors: list[str] = []
|
||||
|
||||
python_requirements: list[tuple[str, str, str]] = []
|
||||
for line in release_requirements.read_text(encoding="utf-8").splitlines():
|
||||
match = re.match(r"^(govoplan-[a-z0-9-]+)\s+@\s+git\+ssh://(.+?\.git)@(v[0-9]+\.[0-9]+\.[0-9]+)$", line.strip())
|
||||
if match:
|
||||
python_requirements.append((match.group(1), f"ssh://{match.group(2)}", match.group(3)))
|
||||
|
||||
if not python_requirements:
|
||||
errors.append("No GovOPlaN git+ssh release requirements found.")
|
||||
|
||||
for package_name, repo_url, tag in python_requirements:
|
||||
if tag != expected_tag:
|
||||
errors.append(f"{package_name} release requirement uses {tag!r}, expected {expected_tag!r}.")
|
||||
version = metadata.version(package_name)
|
||||
if version != tag.removeprefix("v"):
|
||||
errors.append(f"{package_name} installed version {version!r} does not match {tag!r}.")
|
||||
result = subprocess.run(
|
||||
["git", "ls-remote", "--tags", repo_url, f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=45,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0 or f"refs/tags/{tag}" not in result.stdout:
|
||||
errors.append(f"{package_name} tag {tag} is not readable from {repo_url}: {result.stderr.strip()}")
|
||||
|
||||
release_package = json.loads(release_webui.read_text(encoding="utf-8"))
|
||||
if release_package.get("version") != expected_version:
|
||||
errors.append(f"Core release WebUI version is {release_package.get('version')!r}, expected {expected_version!r}.")
|
||||
|
||||
webui_dependencies = release_package.get("dependencies", {})
|
||||
for package_name, spec in sorted(webui_dependencies.items()):
|
||||
if not package_name.startswith("@govoplan/"):
|
||||
continue
|
||||
tag = spec.rsplit("#", 1)[-1] if "#" in spec else ""
|
||||
if tag != expected_tag:
|
||||
errors.append(f"{package_name} release dependency uses {tag!r}, expected {expected_tag}.")
|
||||
package_json = root / "webui" / "node_modules" / package_name / "package.json"
|
||||
if not package_json.exists():
|
||||
errors.append(f"{package_name} is missing from release node_modules.")
|
||||
continue
|
||||
installed = json.loads(package_json.read_text(encoding="utf-8"))
|
||||
if installed.get("version") != expected_version:
|
||||
errors.append(f"{package_name} installed version {installed.get('version')!r}, expected {expected_version!r}.")
|
||||
|
||||
shared_peers = ("lucide-react", "react", "react-dom", "react-router-dom")
|
||||
root_peers = release_package.get("peerDependencies", {})
|
||||
for package_json in sorted((root / "webui" / "node_modules" / "@govoplan").glob("*/package.json")):
|
||||
package = json.loads(package_json.read_text(encoding="utf-8"))
|
||||
peers = package.get("peerDependencies", {})
|
||||
for name in shared_peers:
|
||||
if name in peers and name in root_peers and peers[name] != root_peers[name]:
|
||||
errors.append(f"{package.get('name')} peer {name}={peers[name]!r}, expected {root_peers[name]!r}.")
|
||||
|
||||
if errors:
|
||||
print("\n".join(errors), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"Release refs and metadata passed for {len(python_requirements)} Python packages and {len(webui_dependencies)} WebUI packages.")
|
||||
PY
|
||||
|
||||
run_step "Validate installed module manifests and registry"
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||
|
||||
expected = {
|
||||
"access",
|
||||
"admin",
|
||||
"audit",
|
||||
"calendar",
|
||||
"campaigns",
|
||||
"dashboard",
|
||||
"docs",
|
||||
"files",
|
||||
"identity",
|
||||
"idm",
|
||||
"mail",
|
||||
"ops",
|
||||
"organizations",
|
||||
"policy",
|
||||
"tenancy",
|
||||
}
|
||||
|
||||
manifests = available_module_manifests()
|
||||
missing = sorted(expected - set(manifests))
|
||||
if missing:
|
||||
print(f"Missing installed module manifests: {', '.join(missing)}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
registry = build_platform_registry(tuple(sorted(expected)))
|
||||
snapshot = registry.validate()
|
||||
print("Registry modules:", ", ".join(manifest.id for manifest in snapshot.manifests))
|
||||
PY
|
||||
|
||||
run_step "Run SQLite migration smoke for release modules"
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
|
||||
enabled_modules = (
|
||||
"tenancy",
|
||||
"organizations",
|
||||
"identity",
|
||||
"idm",
|
||||
"access",
|
||||
"admin",
|
||||
"dashboard",
|
||||
"policy",
|
||||
"audit",
|
||||
"campaigns",
|
||||
"files",
|
||||
"mail",
|
||||
"calendar",
|
||||
"docs",
|
||||
"ops",
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-release-migration-") as tmp:
|
||||
database_url = f"sqlite:///{Path(tmp) / 'release-integration.sqlite3'}"
|
||||
result = migrate_database(database_url=database_url, enabled_modules=enabled_modules)
|
||||
if not result.current_revision:
|
||||
raise SystemExit("Migration smoke did not produce an Alembic revision.")
|
||||
print(f"SQLite release migration smoke reached {result.current_revision}.")
|
||||
PY
|
||||
|
||||
run_step "Clone release module test sources"
|
||||
for repo in govoplan-mail govoplan-calendar govoplan-campaign; do
|
||||
git clone --depth 1 --branch "$RELEASE_TAG" "git@git.add-ideas.de:add-ideas/${repo}.git" "$WORK_ROOT/$repo"
|
||||
done
|
||||
|
||||
run_step "Run core backend release tests"
|
||||
"$PYTHON" -m unittest \
|
||||
tests.test_module_system \
|
||||
tests.test_access_contracts \
|
||||
tests.test_identity_organization_contracts \
|
||||
tests.test_core_events \
|
||||
tests.test_policy_contracts
|
||||
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||
|
||||
run_step "Run cloned module backend tests"
|
||||
run_discovered_tests "$WORK_ROOT/govoplan-mail/tests"
|
||||
run_discovered_tests "$WORK_ROOT/govoplan-calendar/tests"
|
||||
run_discovered_tests "$WORK_ROOT/govoplan-campaign/tests"
|
||||
|
||||
run_step "Run core WebUI release tests and build"
|
||||
cd "$ROOT/webui"
|
||||
"$NPM" run test:mail-components
|
||||
"$NPM" run test:module-capabilities
|
||||
"$NPM" run test:module-permutations
|
||||
"$NPM" run build
|
||||
|
||||
run_step "Run cloned module WebUI tests"
|
||||
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-mail/webui"
|
||||
cd "$WORK_ROOT/govoplan-mail/webui"
|
||||
"$NPM" run test:mail-ui
|
||||
|
||||
install_cloned_webui_dependencies "$WORK_ROOT/govoplan-campaign/webui"
|
||||
cd "$WORK_ROOT/govoplan-campaign/webui"
|
||||
"$NPM" run test:policy-ui
|
||||
"$NPM" run test:template-preview
|
||||
"$NPM" run test:import-utils
|
||||
|
||||
echo
|
||||
echo "Release integration check passed."
|
||||
@@ -1,236 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPOS = {
|
||||
"core": ROOT / "src" / "govoplan_core",
|
||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
||||
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||
}
|
||||
PREFIXES = {
|
||||
"core": "govoplan_core",
|
||||
"access": "govoplan_access",
|
||||
"admin": "govoplan_admin",
|
||||
"tenancy": "govoplan_tenancy",
|
||||
"organizations": "govoplan_organizations",
|
||||
"identity": "govoplan_identity",
|
||||
"policy": "govoplan_policy",
|
||||
"audit": "govoplan_audit",
|
||||
"dashboard": "govoplan_dashboard",
|
||||
"files": "govoplan_files",
|
||||
"mail": "govoplan_mail",
|
||||
"campaign": "govoplan_campaign",
|
||||
}
|
||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
||||
WEBUI_REPOS = {
|
||||
"core": ROOT / "webui",
|
||||
"files": ROOT.parent / "govoplan-files" / "webui",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "webui",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "webui",
|
||||
}
|
||||
WEBUI_PACKAGES = {
|
||||
"core": "@govoplan/core-webui",
|
||||
"files": "@govoplan/files-webui",
|
||||
"mail": "@govoplan/mail-webui",
|
||||
"campaign": "@govoplan/campaign-webui",
|
||||
}
|
||||
WEBUI_IMPORT_RE = re.compile(
|
||||
r"(?:from\s+|import\s*\(\s*|require\s*\(\s*|export\s+[^;]*?\s+from\s+)"
|
||||
r"[\"'](?P<package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||
r"|import\s+[\"'](?P<side_effect_package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllowlistedImport:
|
||||
owner: str
|
||||
relative_path: str
|
||||
imported_owner: str
|
||||
reason: str
|
||||
|
||||
|
||||
# Transitional exceptions. This should remain empty; add entries only with a
|
||||
# dated removal plan and a capability/API contract issue.
|
||||
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
|
||||
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Violation:
|
||||
kind: str
|
||||
owner: str
|
||||
path: Path
|
||||
lineno: int
|
||||
imported: str
|
||||
imported_owner: str
|
||||
|
||||
|
||||
def _import_owner(module_name: str) -> str | None:
|
||||
for owner, prefix in PREFIXES.items():
|
||||
if module_name == prefix or module_name.startswith(f"{prefix}."):
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _imports(path: Path) -> list[tuple[int, str]]:
|
||||
tree = ast.parse(path.read_text(), filename=str(path))
|
||||
result: list[tuple[int, str]] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
result.append((node.lineno, alias.name))
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
result.append((node.lineno, node.module))
|
||||
return result
|
||||
|
||||
|
||||
def _relative(owner: str, path: Path) -> str:
|
||||
return path.relative_to(REPOS[owner]).as_posix()
|
||||
|
||||
|
||||
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
|
||||
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
|
||||
|
||||
|
||||
def _violations() -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for owner, root in REPOS.items():
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
for lineno, imported in _imports(path):
|
||||
imported_owner = _import_owner(imported)
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner == "access" and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in PLATFORM_OWNERS and imported_owner == "access":
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner == "access":
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
violations.extend(_webui_violations())
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_package_owner(package_name: str) -> str | None:
|
||||
for owner, package in WEBUI_PACKAGES.items():
|
||||
if package_name == package:
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _webui_source_files(root: Path) -> list[Path]:
|
||||
source_root = root / "src"
|
||||
if not source_root.exists():
|
||||
return []
|
||||
return [
|
||||
path
|
||||
for path in source_root.rglob("*")
|
||||
if path.suffix in {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
|
||||
and "node_modules" not in path.parts
|
||||
and "dist" not in path.parts
|
||||
]
|
||||
|
||||
|
||||
def _webui_dependency_fields(payload: object) -> dict[str, object]:
|
||||
if not isinstance(payload, dict):
|
||||
return {}
|
||||
result: dict[str, object] = {}
|
||||
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, dict):
|
||||
result.update(value)
|
||||
return result
|
||||
|
||||
|
||||
def _webui_package_json_violations(owner: str, root: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
package_paths = [root / "package.json"]
|
||||
if root.name == "webui":
|
||||
package_paths.append(root.parent / "package.json")
|
||||
for package_path in tuple(dict.fromkeys(package_paths)):
|
||||
if not package_path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(package_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
return [Violation("webui-package", owner, package_path, exc.lineno, "invalid package.json", owner)]
|
||||
for package_name in _webui_dependency_fields(payload):
|
||||
imported_owner = _webui_package_owner(str(package_name))
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
if owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-package", owner, package_path, 1, str(package_name), imported_owner))
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_source_violations(owner: str, root: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for path in _webui_source_files(root):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for match in WEBUI_IMPORT_RE.finditer(text):
|
||||
package_name = match.group("package") or match.group("side_effect_package")
|
||||
imported_owner = _webui_package_owner(package_name)
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
lineno = text.count("\n", 0, match.start()) + 1
|
||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_violations() -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for owner, root in WEBUI_REPOS.items():
|
||||
if not root.exists():
|
||||
continue
|
||||
violations.extend(_webui_package_json_violations(owner, root))
|
||||
violations.extend(_webui_source_violations(owner, root))
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
violations = _violations()
|
||||
if not violations:
|
||||
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
|
||||
return 0
|
||||
|
||||
print("Dependency boundary violations:")
|
||||
for item in violations:
|
||||
print(
|
||||
f"- {item.path}:{item.lineno}: {item.owner} {item.kind} imports {item.imported} "
|
||||
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
|
||||
)
|
||||
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
|
||||
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
|
||||
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
|
||||
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
|
||||
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
|
||||
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
|
||||
args = parser.parse_args()
|
||||
|
||||
private_path = args.private_key.expanduser()
|
||||
public_path = args.public_key.expanduser() if args.public_key else None
|
||||
keyring_path = args.keyring.expanduser() if args.keyring else None
|
||||
if private_path.exists() and not args.force:
|
||||
raise SystemExit(f"Private key already exists: {private_path}")
|
||||
if public_path is not None and public_path.exists() and not args.force:
|
||||
raise SystemExit(f"Public key already exists: {public_path}")
|
||||
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
private_bytes = private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
public_base64 = base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
private_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
private_path.write_bytes(private_bytes)
|
||||
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
if public_path is not None:
|
||||
public_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
public_path.write_text(public_base64 + "\n", encoding="utf-8")
|
||||
|
||||
if keyring_path is not None:
|
||||
keyring_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring = {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": args.key_id,
|
||||
"status": args.status,
|
||||
"public_key": public_base64,
|
||||
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
],
|
||||
}
|
||||
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"private_key={private_path}")
|
||||
if public_path is not None:
|
||||
print(f"public_key={public_path}")
|
||||
if keyring_path is not None:
|
||||
print(f"keyring={keyring_path}")
|
||||
print(f"key_id={args.key_id}")
|
||||
print(f"public_key_base64={public_base64}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,405 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate and sign a GovOPlaN module package release catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CatalogModule:
|
||||
module_id: str
|
||||
repo: str
|
||||
python_package: str
|
||||
name: str
|
||||
description: str
|
||||
tags: tuple[str, ...]
|
||||
webui_package: str | None = None
|
||||
provides_interfaces: tuple[dict[str, object], ...] = ()
|
||||
requires_interfaces: tuple[dict[str, object], ...] = ()
|
||||
|
||||
|
||||
CATALOG_MODULES = (
|
||||
CatalogModule(
|
||||
module_id="tenancy",
|
||||
repo="govoplan-tenancy",
|
||||
python_package="govoplan-tenancy",
|
||||
name="Tenancy",
|
||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="organizations",
|
||||
repo="govoplan-organizations",
|
||||
python_package="govoplan-organizations",
|
||||
name="Organizations",
|
||||
description="Organization units, functions, and account-held function assignments.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="identity",
|
||||
repo="govoplan-identity",
|
||||
python_package="govoplan-identity",
|
||||
name="Identity",
|
||||
description="Canonical identities and links between identities and platform accounts.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="access",
|
||||
repo="govoplan-access",
|
||||
python_package="govoplan-access",
|
||||
name="Access",
|
||||
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/access-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="admin",
|
||||
repo="govoplan-admin",
|
||||
python_package="govoplan-admin",
|
||||
name="Admin",
|
||||
description="System settings, governance templates, module management, and admin shell contributions.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/admin-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="policy",
|
||||
repo="govoplan-policy",
|
||||
python_package="govoplan-policy",
|
||||
name="Policy",
|
||||
description="Policy and governance capability module.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/policy-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="audit",
|
||||
repo="govoplan-audit",
|
||||
python_package="govoplan-audit",
|
||||
name="Audit",
|
||||
description="Audit-log storage and audit administration routes.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/audit-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="dashboard",
|
||||
repo="govoplan-dashboard",
|
||||
python_package="govoplan-dashboard",
|
||||
name="Dashboard",
|
||||
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/dashboard-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="files",
|
||||
repo="govoplan-files",
|
||||
python_package="govoplan-files",
|
||||
name="Files",
|
||||
description="Managed file spaces and campaign attachment integration.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/files-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="mail",
|
||||
repo="govoplan-mail",
|
||||
python_package="govoplan-mail",
|
||||
name="Mail",
|
||||
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/mail-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="campaigns",
|
||||
repo="govoplan-campaign",
|
||||
python_package="govoplan-campaign",
|
||||
name="Campaigns",
|
||||
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
||||
tags=("official", "business-module"),
|
||||
webui_package="@govoplan/campaign-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="calendar",
|
||||
repo="govoplan-calendar",
|
||||
python_package="govoplan-calendar",
|
||||
name="Calendar",
|
||||
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/calendar-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="docs",
|
||||
repo="govoplan-docs",
|
||||
python_package="govoplan-docs",
|
||||
name="Docs",
|
||||
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/docs-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="ops",
|
||||
repo="govoplan-ops",
|
||||
python_package="govoplan-ops",
|
||||
name="Ops",
|
||||
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/ops-webui",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
||||
parser.add_argument("--channel", default="stable")
|
||||
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
||||
parser.add_argument("--expires-days", type=int, default=90)
|
||||
parser.add_argument("--catalog-output", type=Path, required=True)
|
||||
parser.add_argument("--keyring-output", type=Path)
|
||||
parser.add_argument(
|
||||
"--catalog-signing-key",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="KEY_ID=PRIVATE_KEY",
|
||||
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
||||
)
|
||||
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
||||
parser.add_argument("--repository-base", default=GITEA_BASE)
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version.removeprefix("v")
|
||||
tag = f"v{version}"
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
expires_at = generated_at + timedelta(days=args.expires_days)
|
||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||
|
||||
catalog = _catalog_payload(
|
||||
version=version,
|
||||
tag=tag,
|
||||
channel=args.channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
expires_at=expires_at,
|
||||
repository_base=args.repository_base.rstrip("/"),
|
||||
public_base_url=args.public_base_url.rstrip("/"),
|
||||
)
|
||||
if signing_keys:
|
||||
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
||||
|
||||
output = args.catalog_output.expanduser()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
if args.keyring_output is not None:
|
||||
keyring_output = args.keyring_output.expanduser()
|
||||
keyring_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring_output.write_text(
|
||||
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"catalog={output}")
|
||||
if args.keyring_output is not None:
|
||||
print(f"keyring={args.keyring_output.expanduser()}")
|
||||
print(f"channel={args.channel}")
|
||||
print(f"sequence={sequence}")
|
||||
print(f"version={version}")
|
||||
return 0
|
||||
|
||||
|
||||
def _catalog_payload(
|
||||
*,
|
||||
version: str,
|
||||
tag: str,
|
||||
channel: str,
|
||||
sequence: int,
|
||||
generated_at: datetime,
|
||||
expires_at: datetime,
|
||||
repository_base: str,
|
||||
public_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
manifests = _discovered_catalog_manifests()
|
||||
modules: list[dict[str, Any]] = []
|
||||
for module in CATALOG_MODULES:
|
||||
manifest = manifests.get(module.module_id)
|
||||
module_version = manifest.version if manifest is not None else version
|
||||
module_tag = f"v{module_version.removeprefix('v')}"
|
||||
entry: dict[str, Any] = {
|
||||
"module_id": module.module_id,
|
||||
"name": module.name,
|
||||
"description": module.description,
|
||||
"version": module_version,
|
||||
"action": "install",
|
||||
"python_package": module.python_package,
|
||||
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
|
||||
"license_features": [f"module.{module.module_id}"],
|
||||
"tags": list(module.tags),
|
||||
}
|
||||
if module.webui_package:
|
||||
entry["webui_package"] = module.webui_package
|
||||
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
|
||||
manifest_metadata = _manifest_catalog_metadata(manifest)
|
||||
entry.update(manifest_metadata)
|
||||
if module.provides_interfaces:
|
||||
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
|
||||
if module.requires_interfaces:
|
||||
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
|
||||
modules.append(entry)
|
||||
|
||||
return {
|
||||
"catalog_version": "1",
|
||||
"channel": channel,
|
||||
"sequence": sequence,
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"expires_at": _json_datetime(expires_at),
|
||||
"release": {
|
||||
"version": version,
|
||||
"tag": tag,
|
||||
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
||||
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
||||
},
|
||||
"core_release": {
|
||||
"name": "GovOPlaN Core",
|
||||
"version": version,
|
||||
"python_package": "govoplan-core",
|
||||
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
||||
"webui_package": "@govoplan/core-webui",
|
||||
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
||||
},
|
||||
"modules": modules,
|
||||
}
|
||||
|
||||
|
||||
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
|
||||
try:
|
||||
return available_module_manifests(ignore_load_errors=True)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _manifest_catalog_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
|
||||
if manifest is None:
|
||||
return {}
|
||||
payload: dict[str, object] = {}
|
||||
if manifest.dependencies:
|
||||
payload["dependencies"] = list(manifest.dependencies)
|
||||
if manifest.optional_dependencies:
|
||||
payload["optional_dependencies"] = list(manifest.optional_dependencies)
|
||||
if manifest.migration_spec is not None:
|
||||
payload["migration_safety"] = "requires_review"
|
||||
payload["migration_notes"] = "Module owns database migrations; review release notes and migration output before activation."
|
||||
if manifest.migration_spec.migration_after:
|
||||
payload["migration_after"] = list(manifest.migration_spec.migration_after)
|
||||
if manifest.migration_spec.migration_before:
|
||||
payload["migration_before"] = list(manifest.migration_spec.migration_before)
|
||||
if manifest.migration_spec.migration_tasks:
|
||||
tasks: list[dict[str, object]] = []
|
||||
for task in manifest.migration_spec.migration_tasks:
|
||||
task_payload: dict[str, object] = {
|
||||
"task_id": task.task_id,
|
||||
"phase": task.phase,
|
||||
"summary": task.summary,
|
||||
"task_version": task.task_version,
|
||||
"safety": task.safety,
|
||||
"idempotent": task.idempotent,
|
||||
}
|
||||
if task.timeout_seconds is not None:
|
||||
task_payload["timeout_seconds"] = task.timeout_seconds
|
||||
tasks.append(task_payload)
|
||||
payload["migration_tasks"] = tasks
|
||||
if manifest.provides_interfaces:
|
||||
payload["provides_interfaces"] = [
|
||||
{"name": item.name, "version": item.version}
|
||||
for item in manifest.provides_interfaces
|
||||
]
|
||||
if manifest.requires_interfaces:
|
||||
requirements: list[dict[str, object]] = []
|
||||
for item in manifest.requires_interfaces:
|
||||
requirement: dict[str, object] = {
|
||||
"name": item.name,
|
||||
"optional": item.optional,
|
||||
}
|
||||
if item.version_min is not None:
|
||||
requirement["version_min"] = item.version_min
|
||||
if item.version_max_exclusive is not None:
|
||||
requirement["version_max_exclusive"] = item.version_max_exclusive
|
||||
requirements.append(requirement)
|
||||
payload["requires_interfaces"] = requirements
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
||||
key_id, separator, path_text = value.partition("=")
|
||||
if not separator or not key_id.strip() or not path_text.strip():
|
||||
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||
path = Path(path_text).expanduser()
|
||||
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(private_key, Ed25519PrivateKey):
|
||||
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||
return key_id.strip(), private_key
|
||||
|
||||
|
||||
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
|
||||
signature_payload = dict(payload)
|
||||
signature_payload.pop("signature", None)
|
||||
signature_payload.pop("signatures", None)
|
||||
signature = private_key.sign(_canonical_bytes(signature_payload))
|
||||
return {
|
||||
"algorithm": "ed25519",
|
||||
"key_id": key_id,
|
||||
"value": base64.b64encode(signature).decode("ascii"),
|
||||
}
|
||||
|
||||
|
||||
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
|
||||
return {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": key_id,
|
||||
"status": "active",
|
||||
"public_key": _public_key_base64(private_key),
|
||||
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
for key_id, private_key in signing_keys
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
return base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _canonical_bytes(payload: object) -> bytes:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def _json_datetime(value: datetime) -> str:
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,107 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/generate-release-lock.sh [options]
|
||||
|
||||
Generates webui/package-lock.release.json from webui/package.release.json in a
|
||||
temporary workspace. The normal development package.json and package-lock.json
|
||||
are left untouched.
|
||||
|
||||
Run this after the module git tags referenced by package.release.json exist and
|
||||
are reachable, and before tagging the core release commit that should contain
|
||||
the regenerated release lockfile.
|
||||
|
||||
Options:
|
||||
--npm <path> npm executable to use.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WEBUI="$ROOT/webui"
|
||||
NPM_BIN="${NPM:-}"
|
||||
NODE_BIN="${NODE:-}"
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
if [[ -z "$NODE_BIN" ]]; then
|
||||
if [[ -x "$(dirname "$NPM_BIN")/node" ]]; then
|
||||
NODE_BIN="$(dirname "$NPM_BIN")/node"
|
||||
else
|
||||
NODE_BIN="node"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$WEBUI/package.release.json" ]] || fail "missing $WEBUI/package.release.json"
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm executable not found: $NPM_BIN"
|
||||
command -v "$NODE_BIN" >/dev/null 2>&1 || fail "node executable not found: $NODE_BIN"
|
||||
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-lock.XXXXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"
|
||||
if [[ -f "$WEBUI/package-lock.release.json" ]]; then
|
||||
cp "$WEBUI/package-lock.release.json" "$TMP_DIR/package-lock.json"
|
||||
fi
|
||||
|
||||
echo "Generating release lockfile from $WEBUI/package.release.json"
|
||||
echo "Temporary workspace: $TMP_DIR"
|
||||
(
|
||||
cd "$TMP_DIR"
|
||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||
mapfile -t GIT_PACKAGES < <(
|
||||
PATH="$(dirname "$NODE_BIN"):$PATH" "$NODE_BIN" <<'NODE'
|
||||
const fs = require("fs");
|
||||
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||
for (const group of ["dependencies", "devDependencies", "optionalDependencies"]) {
|
||||
for (const [name, spec] of Object.entries(pkg[group] || {})) {
|
||||
if (typeof spec === "string" && spec.startsWith("git+")) {
|
||||
console.log(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
NODE
|
||||
)
|
||||
if [[ "${#GIT_PACKAGES[@]}" -gt 0 ]]; then
|
||||
echo "Refreshing git package lock entries: ${GIT_PACKAGES[*]}"
|
||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" update --package-lock-only --ignore-scripts "${GIT_PACKAGES[@]}"
|
||||
fi
|
||||
)
|
||||
|
||||
cp "$TMP_DIR/package-lock.json" "$WEBUI/package-lock.release.json"
|
||||
echo "Updated $WEBUI/package-lock.release.json"
|
||||
@@ -1,580 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import markdown backlog/planning files into Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||
|
||||
REPO_ROOTS = {
|
||||
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
||||
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
||||
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
||||
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
||||
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
||||
"web": pathlib.Path("/mnt/DATA/git/govoplan-web"),
|
||||
}
|
||||
|
||||
MODULE_LABELS = {
|
||||
"core": "module/core",
|
||||
"access": "module/access",
|
||||
"mail": "module/mail",
|
||||
"files": "module/files",
|
||||
"campaign": "module/campaign",
|
||||
"web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
candidates = list(build_candidates(args))
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
print_preview(candidates)
|
||||
return 0
|
||||
|
||||
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
||||
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
||||
skipped = len(candidates) - len(missing)
|
||||
|
||||
print(f"Skipped {skipped} existing issue candidate(s).")
|
||||
print(f"Missing {len(missing)} issue candidate(s).")
|
||||
print_preview(missing)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
created_by_repo: dict[str, list[str]] = {}
|
||||
for candidate in missing:
|
||||
state = states[candidate.repo_key]
|
||||
validate_labels(candidate, state)
|
||||
issue = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": [state.label_ids[label] for label in candidate.labels],
|
||||
},
|
||||
)
|
||||
number = issue.get("number") or issue.get("index")
|
||||
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
|
||||
print("Created issues:")
|
||||
for repo_key, titles in created_by_repo.items():
|
||||
print(f"{repo_key}:")
|
||||
for title in titles:
|
||||
print(f" {title}")
|
||||
if not created_by_repo:
|
||||
print(" none")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||
if args.product_backlog.exists():
|
||||
yield from parse_product_backlog(args.product_backlog)
|
||||
if args.access_plan and args.access_plan.exists():
|
||||
yield from parse_access_plan(args.access_plan)
|
||||
if args.web_todo.exists():
|
||||
yield from parse_simple_todo(args.web_todo)
|
||||
|
||||
|
||||
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
headings: list[tuple[int, str]] = []
|
||||
status = ""
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
if level == 2 and title == "Completed Work Archive":
|
||||
break
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
status = ""
|
||||
continue
|
||||
|
||||
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
||||
if status_match:
|
||||
status = status_match.group(1).strip()
|
||||
continue
|
||||
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
section = section_title(headings)
|
||||
repo_key, labels = classify_product_item(item, section)
|
||||
prefix = type_prefix(labels)
|
||||
title = format_title(prefix, item)
|
||||
source = str(path)
|
||||
body = body_for_item(
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
section=section,
|
||||
status=status,
|
||||
item=item,
|
||||
note="Imported from the consolidated GovOPlaN product backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=tuple(sorted(set(labels))),
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
yield from access_stage_candidates(path, lines)
|
||||
yield from access_immediate_candidates(path, lines)
|
||||
yield from access_decision_candidates(path, lines)
|
||||
|
||||
|
||||
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
||||
if not match:
|
||||
index += 1
|
||||
continue
|
||||
stage = match.group(1)
|
||||
name = match.group(2).strip()
|
||||
start_line = index + 1
|
||||
index += 1
|
||||
block: list[str] = []
|
||||
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
||||
block.append(lines[index])
|
||||
index += 1
|
||||
item = f"Access extraction Stage {stage}: {name}"
|
||||
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from the GovOPlaN access extraction plan.",
|
||||
"",
|
||||
f"- Source: `{path}`",
|
||||
f"- Line: `{start_line}`",
|
||||
f"- Section: `Stage {stage}: {name}`",
|
||||
"",
|
||||
"Plan excerpt:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"### Stage {stage}: {name}",
|
||||
*block,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=start_line,
|
||||
)
|
||||
|
||||
|
||||
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Immediate Backlog":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
title_item = access_title_item(item)
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Immediate Backlog",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", title_item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Open Decisions":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Open Decisions",
|
||||
status="decision needed",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", f"Decide: {item}"),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
||||
yield Candidate(
|
||||
repo_key="web",
|
||||
title=format_title("[Task]", item),
|
||||
body=body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="TODO",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN website TODO file.",
|
||||
),
|
||||
labels=labels("web", "type/task", "area/marketing"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
||||
text = f"{section} {item}".lower()
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
issue_type = "type/task"
|
||||
|
||||
if "dsar search/export/delete" in text:
|
||||
return "core", labels("core", "type/feature", "area/governance")
|
||||
if "fully stream zip uploads" in text:
|
||||
return "files", labels("files", "type/task", "area/api")
|
||||
if "installer packaging approach" in text:
|
||||
return "core", labels("core", "type/task", "area/devex")
|
||||
if "profile reselection/revalidation" in text:
|
||||
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
||||
if "module boundaries" in text:
|
||||
return "core", labels("core", issue_type, "area/module-system")
|
||||
if "external storage connectors" in text:
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
elif "controlled sending" in text:
|
||||
repo_key = "campaign"
|
||||
area = "area/api"
|
||||
elif "mail profiles" in text:
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
|
||||
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
if "external storage connectors" not in text and any(
|
||||
word in text
|
||||
for word in (
|
||||
"campaign",
|
||||
"recipient",
|
||||
"attachment",
|
||||
"zip",
|
||||
"evidence",
|
||||
"wizard",
|
||||
"send",
|
||||
"archive",
|
||||
"message preview",
|
||||
"address book",
|
||||
"carddav",
|
||||
"mailing lists",
|
||||
)
|
||||
):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
||||
repo_key = "core"
|
||||
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
||||
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
||||
repo_key = "core"
|
||||
area = "area/module-system"
|
||||
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
||||
repo_key = "core"
|
||||
area = "area/release"
|
||||
if repo_key == "core" and not any(
|
||||
phrase in text
|
||||
for phrase in (
|
||||
"external storage connectors",
|
||||
"controlled sending",
|
||||
"report, evidence",
|
||||
"recipient import",
|
||||
"campaign and attachment",
|
||||
)
|
||||
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
||||
area = "area/docs"
|
||||
if "ui lists often start at index 2" in text:
|
||||
issue_type = "type/bug"
|
||||
area = "area/webui"
|
||||
repo_key = "core"
|
||||
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
||||
issue_type = "type/feature"
|
||||
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
||||
issue_type = "type/debt"
|
||||
|
||||
return repo_key, labels(repo_key, issue_type, area)
|
||||
|
||||
|
||||
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
||||
parts = [initial.strip()]
|
||||
next_index = index + 1
|
||||
while next_index < len(lines):
|
||||
line = lines[next_index]
|
||||
if not line.startswith((" ", "\t")):
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
break
|
||||
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
||||
break
|
||||
parts.append(stripped)
|
||||
next_index += 1
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
||||
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
||||
base.update(extra)
|
||||
if "status/needs-info" in base:
|
||||
base.discard("status/triage")
|
||||
return tuple(sorted(base))
|
||||
|
||||
|
||||
def type_prefix(label_set: Iterable[str]) -> str:
|
||||
labels_set = set(label_set)
|
||||
if "type/bug" in labels_set:
|
||||
return "[Bug]"
|
||||
if "type/feature" in labels_set:
|
||||
return "[Feature]"
|
||||
if "type/debt" in labels_set:
|
||||
return "[Debt]"
|
||||
if "type/docs" in labels_set:
|
||||
return "[Docs]"
|
||||
return "[Task]"
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
||||
title = f"{prefix} {clean}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def access_title_item(item: str) -> str:
|
||||
lowered = item.lower()
|
||||
if "compatibility wrappers" in lowered:
|
||||
return "Wire core auth compatibility through access capabilities"
|
||||
return item
|
||||
|
||||
|
||||
def body_for_item(
|
||||
*,
|
||||
fingerprint: str,
|
||||
source: str,
|
||||
line: int,
|
||||
section: str,
|
||||
status: str,
|
||||
item: str,
|
||||
note: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
note,
|
||||
"",
|
||||
f"- Source: `{source}`",
|
||||
f"- Line: `{line}`",
|
||||
f"- Section: `{section}`",
|
||||
]
|
||||
if status:
|
||||
lines.append(f"- Source status: `{status}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"Imported backlog item:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"- [ ] {item}",
|
||||
"```",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([source, str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
||||
grouped: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
||||
return grouped
|
||||
|
||||
|
||||
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in repo_keys:
|
||||
root = repo_root(REPO_ROOTS[repo_key])
|
||||
target = infer_target(root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
if issue.get("state") == "closed" and "Moved to `" in body:
|
||||
continue
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
||||
return states
|
||||
|
||||
|
||||
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
||||
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
||||
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}:")
|
||||
for candidate in grouped[:limit]:
|
||||
print(f" {candidate.title}")
|
||||
if len(grouped) > limit:
|
||||
print(f" ... {len(grouped) - limit} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post a standardized Codex state update comment to a Gitea issue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
STATUS_LABELS = {
|
||||
"started": "status/in-progress",
|
||||
"progress": "status/in-progress",
|
||||
"blocked": "status/blocked",
|
||||
"needs-info": "status/needs-info",
|
||||
"ready": "status/ready",
|
||||
"done": "",
|
||||
"note": "",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
||||
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
||||
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
||||
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
||||
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
||||
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
||||
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
||||
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
||||
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
body = build_body(args)
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Issue: #{args.issue}")
|
||||
print(body)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
||||
body={"body": body},
|
||||
)
|
||||
if args.close:
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print("Posted Codex note.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_body(args: argparse.Namespace) -> str:
|
||||
lines = [f"## Codex State: {args.status}", ""]
|
||||
append_section(lines, "Summary", args.summary)
|
||||
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
||||
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
||||
append_section(lines, "Next / Blocked", args.next)
|
||||
if STATUS_LABELS[args.status]:
|
||||
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
||||
if args.body_file:
|
||||
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
||||
if not items:
|
||||
return
|
||||
lines.extend([f"### {title}", ""])
|
||||
lines.extend(f"- {item}" for item in items)
|
||||
lines.append("")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,597 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
EXCLUDED_FILE_PATTERNS = (
|
||||
"/.git/",
|
||||
"/.venv/",
|
||||
"/venv/",
|
||||
"/site-packages/",
|
||||
"/node_modules/",
|
||||
"/__pycache__/",
|
||||
"/dist/",
|
||||
"/build/",
|
||||
"/.cache/",
|
||||
"/.module-test-build/",
|
||||
)
|
||||
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
||||
ACTION_RE = re.compile(
|
||||
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
||||
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
||||
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SKIP_SECTION_RE = re.compile(
|
||||
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
||||
r"available now|done|archive)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ACTIVE_SECTION_RE = re.compile(
|
||||
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
||||
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
GENERIC_LABELS = [
|
||||
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
||||
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
||||
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
||||
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
||||
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
||||
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
||||
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
||||
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
||||
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
||||
("priority/p2", "fbca04", "Normal planned priority."),
|
||||
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
||||
]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
remote: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SourceFile:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
source_kind: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
priority: str
|
||||
milestone: str
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
candidates = list(build_candidates(sources))
|
||||
candidates = dedupe_candidates(candidates)
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Backlog-like source files: {len(sources)}")
|
||||
print(f"Issue candidates: {len(candidates)}")
|
||||
for repo_key, count in grouped_counts(candidates).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print_preview(candidates)
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
return 0
|
||||
|
||||
missing_by_repo: dict[str, list[Candidate]] = {}
|
||||
skipped = 0
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
||||
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
||||
for candidate in candidates:
|
||||
state = states[candidate.repo_key]
|
||||
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
||||
skipped += 1
|
||||
continue
|
||||
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
|
||||
missing_total = sum(len(items) for items in missing_by_repo.values())
|
||||
print(f"Skipped existing: {skipped}")
|
||||
print(f"Missing candidates: {missing_total}")
|
||||
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
for repo_key, repo_candidates in missing_by_repo.items():
|
||||
state = states[repo_key]
|
||||
for candidate in repo_candidates:
|
||||
label_ids = [state.label_ids[label] for label in candidate.labels]
|
||||
if candidate.priority not in candidate.labels:
|
||||
label_ids.append(state.label_ids[candidate.priority])
|
||||
milestone_id = ensure_milestone(state, candidate.milestone)
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": label_ids,
|
||||
"milestone": milestone_id,
|
||||
},
|
||||
)
|
||||
number = created.get("number") or created.get("index")
|
||||
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
milestone_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
remote = result.stdout.strip()
|
||||
if "git.add-ideas.de" not in remote:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
||||
sources: list[SourceFile] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
||||
continue
|
||||
path_text = path.as_posix()
|
||||
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
||||
continue
|
||||
if is_excluded_repo_file(path):
|
||||
continue
|
||||
if is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
||||
|
||||
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.path.resolve())] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
||||
|
||||
|
||||
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||
text = path.as_posix()
|
||||
name = path.name.lower()
|
||||
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
|
||||
return True
|
||||
if "testing_plan" in name or "test_plan" in name:
|
||||
return True
|
||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||
return True
|
||||
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/govoplan-web/TODO.md"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_backlog(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
if b"\x00" in chunk:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
||||
for source in sources:
|
||||
suffix = source.path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
yield from parse_csv_source(source)
|
||||
else:
|
||||
yield from parse_text_source(source)
|
||||
|
||||
|
||||
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for index, row in enumerate(reader, start=2):
|
||||
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
||||
if not story:
|
||||
continue
|
||||
identifier = (row.get("id") or "").strip()
|
||||
priority = normalize_priority(row.get("priority") or "")
|
||||
milestone = (row.get("epic") or "Backlog").strip()
|
||||
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
||||
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a CSV backlog file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{index}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Milestone: `{milestone}`",
|
||||
"",
|
||||
"CSV row:",
|
||||
"",
|
||||
"```text",
|
||||
", ".join(f"{key}={value}" for key, value in row.items()),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=index,
|
||||
)
|
||||
|
||||
|
||||
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
try:
|
||||
lines = source.path.read_text(encoding="utf-8").splitlines()
|
||||
except UnicodeDecodeError:
|
||||
lines = source.path.read_text(encoding="latin-1").splitlines()
|
||||
|
||||
headings: list[tuple[int, str]] = []
|
||||
in_tasks = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading = parse_heading(line)
|
||||
if heading:
|
||||
level, title = heading
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
in_tasks = False
|
||||
continue
|
||||
|
||||
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
||||
in_tasks = True
|
||||
continue
|
||||
|
||||
item = parse_open_item(line)
|
||||
if item is None and is_action_context(source.path, headings, in_tasks):
|
||||
item = parse_plain_action_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
if not item or should_skip_item(item, headings):
|
||||
continue
|
||||
|
||||
section = section_title(headings)
|
||||
priority = priority_from_context(section, item)
|
||||
milestone = milestone_from_context(source.path, section)
|
||||
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
||||
title = format_title(prefix, item)
|
||||
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a backlog-like text file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{line_number}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Section: `{section or 'none'}`",
|
||||
"",
|
||||
"Imported item:",
|
||||
"",
|
||||
"```text",
|
||||
item,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_heading(line: str) -> tuple[int, str] | None:
|
||||
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
||||
if markdown:
|
||||
return len(markdown.group(1)), clean_text(markdown.group(2))
|
||||
underlined = re.match(r"^(.+?)\s*$", line)
|
||||
if underlined and line.strip() and len(line.strip()) < 120:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_open_item(line: str) -> str | None:
|
||||
patterns = [
|
||||
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
return clean_text(match.group("text"))
|
||||
return None
|
||||
|
||||
|
||||
def parse_plain_action_item(line: str) -> str | None:
|
||||
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
text = clean_text(match.group("text"))
|
||||
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
||||
return None
|
||||
if ACTION_RE.match(text) or is_short_noun_task(text):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
||||
if in_tasks:
|
||||
return True
|
||||
context = section_title(headings)
|
||||
if not context:
|
||||
return False
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return False
|
||||
if ACTIVE_SECTION_RE.search(context):
|
||||
return True
|
||||
return path.name.lower() in {"todo.txt", "todo.md"}
|
||||
|
||||
|
||||
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
||||
context = section_title(headings)
|
||||
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
||||
return True
|
||||
if item.endswith((",", ";")):
|
||||
return True
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return True
|
||||
if len(item) < 4:
|
||||
return True
|
||||
if item.endswith(":") and len(item.split()) <= 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_short_noun_task(text: str) -> bool:
|
||||
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
text = text.strip()
|
||||
text = re.sub(r"^☐\s*", "", text)
|
||||
text = re.sub(r"^✅\s*", "", text)
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip(" -")
|
||||
|
||||
|
||||
def leading_width(indent: str) -> int:
|
||||
return len(indent.replace("\t", " "))
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def priority_from_context(section: str, item: str) -> str:
|
||||
text = f"{section} {item}".lower()
|
||||
if "p0" in text or "critical path" in text:
|
||||
return "priority/p0"
|
||||
if "p1" in text or "immediate" in text or "next sprint" in text:
|
||||
return "priority/p1"
|
||||
if "p2" in text:
|
||||
return "priority/p2"
|
||||
if "p3" in text or "future" in text or "later" in text:
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def normalize_priority(value: str) -> str:
|
||||
value = value.strip().lower()
|
||||
if value == "p0":
|
||||
return "priority/p0"
|
||||
if value == "p1":
|
||||
return "priority/p1"
|
||||
if value == "p2":
|
||||
return "priority/p2"
|
||||
if value == "p3":
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
||||
if section:
|
||||
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
||||
if parts:
|
||||
return parts[-1][:120]
|
||||
return path.stem.replace("_", " ").replace("-", " ").title()
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
unique: list[Candidate] = []
|
||||
for candidate in candidates:
|
||||
key = (candidate.repo_key, normalize_title(candidate.title))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for candidate in candidates:
|
||||
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
if apply:
|
||||
for name, color, description in GENERIC_LABELS:
|
||||
if name in label_ids:
|
||||
continue
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
||||
)
|
||||
label_ids[str(created["name"])] = int(created["id"])
|
||||
print(f"created label {target.owner}/{target.repo}:{name}")
|
||||
|
||||
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
||||
if missing and apply:
|
||||
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
||||
|
||||
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
||||
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
||||
|
||||
|
||||
def ensure_milestone(state: RepoState, title: str) -> int:
|
||||
if title in state.milestone_ids:
|
||||
return state.milestone_ids[title]
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
||||
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
||||
)
|
||||
state.milestone_ids[title] = int(created["id"])
|
||||
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
||||
return state.milestone_ids[title]
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
||||
by_repo: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
for repo_key, items in sorted(by_repo.items()):
|
||||
print(f"{repo_key}:")
|
||||
for item in items[:limit_per_repo]:
|
||||
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
||||
if len(items) > limit_per_repo:
|
||||
print(f" ... {len(items) - limit_per_repo} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install Gitea issue workflow files into one or more repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaError, repo_root
|
||||
|
||||
|
||||
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
WORKFLOW_FILES = (
|
||||
".gitea/PULL_REQUEST_TEMPLATE.md",
|
||||
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
||||
".gitea/ISSUE_TEMPLATE/config.yaml",
|
||||
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
||||
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
||||
".gitea/ISSUE_TEMPLATE/task.md",
|
||||
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
||||
)
|
||||
MODULE_LABEL_BY_REPO = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
||||
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
||||
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
||||
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
||||
unique_roots = list(dict.fromkeys(target_roots))
|
||||
if args.module_label and len(unique_roots) != 1:
|
||||
raise GiteaError("--module-label can only be used with a single target")
|
||||
|
||||
rel_paths = list(WORKFLOW_FILES)
|
||||
if args.include_labels_file:
|
||||
rel_paths.append("docs/gitea-labels.json")
|
||||
|
||||
for target_root in unique_roots:
|
||||
module_label = args.module_label or infer_module_label(target_root)
|
||||
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to write files.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def infer_module_label(target_root: pathlib.Path) -> str:
|
||||
repo_name = target_root.name
|
||||
try:
|
||||
repo_name = target_root.resolve().name
|
||||
except OSError:
|
||||
pass
|
||||
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
||||
if label:
|
||||
return label
|
||||
if repo_name.startswith("govoplan-"):
|
||||
suffix = repo_name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/<name>.")
|
||||
|
||||
|
||||
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
||||
print(f"Target: {target_root} ({module_label})")
|
||||
for rel_path in rel_paths:
|
||||
source = SOURCE_ROOT / rel_path
|
||||
target = target_root / rel_path
|
||||
if not source.exists():
|
||||
raise GiteaError(f"missing workflow source file: {source}")
|
||||
|
||||
content = source.read_text(encoding="utf-8")
|
||||
content = transform_content(rel_path, content, module_label)
|
||||
|
||||
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
||||
if existing == content:
|
||||
print(f"unchanged {rel_path}")
|
||||
continue
|
||||
|
||||
action = "create" if existing is None else "update"
|
||||
print(f"{action if apply else 'would ' + action} {rel_path}")
|
||||
if apply:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
||||
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
||||
return content.replace(" - module/core", f" - {module_label}")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Move issue labels from repository-local labels to organization labels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||
parser.add_argument(
|
||||
"--issue-mode",
|
||||
choices=("replace", "delete-add"),
|
||||
default="replace",
|
||||
help="replace labels in one request, or delete local ids before adding org ids",
|
||||
)
|
||||
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
owner = args.org or target.owner
|
||||
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||
if client.token is None:
|
||||
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||
|
||||
repo_filter = re.compile(args.repo_regex)
|
||||
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||
if not org_labels:
|
||||
raise GiteaError(f"{owner} has no organization labels")
|
||||
|
||||
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||
totals = Totals()
|
||||
print(f"Organization: {owner}")
|
||||
print(f"Repositories: {len(repos)}")
|
||||
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||
|
||||
for index, repo in enumerate(repos, start=1):
|
||||
repo_result = migrate_repository(
|
||||
client,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
org_labels=org_labels,
|
||||
issue_mode=args.issue_mode,
|
||||
apply=args.apply,
|
||||
delete_repo_labels=args.delete_repo_labels,
|
||||
)
|
||||
totals.add(repo_result)
|
||||
if repo_result.has_work:
|
||||
print(
|
||||
f"[{index}/{len(repos)}] {repo}: "
|
||||
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||
f"errors={len(repo_result.errors)}"
|
||||
)
|
||||
for error in repo_result.errors:
|
||||
print(f" error: {error}")
|
||||
else:
|
||||
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||
|
||||
print("Summary:")
|
||||
print(f" repositories processed: {totals.repositories}")
|
||||
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||
print(f" errors: {totals.errors}")
|
||||
if totals.errors:
|
||||
return 1
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class RepoResult:
|
||||
def __init__(self) -> None:
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors: list[str] = []
|
||||
self.matched_local_labels = 0
|
||||
|
||||
@property
|
||||
def has_work(self) -> bool:
|
||||
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||
|
||||
|
||||
class Totals:
|
||||
def __init__(self) -> None:
|
||||
self.repositories = 0
|
||||
self.repositories_with_work = 0
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors = 0
|
||||
|
||||
def add(self, result: RepoResult) -> None:
|
||||
self.repositories += 1
|
||||
if result.has_work:
|
||||
self.repositories_with_work += 1
|
||||
self.issue_label_migrations += result.issue_label_migrations
|
||||
self.repo_label_deletions += result.repo_label_deletions
|
||||
self.errors += len(result.errors)
|
||||
|
||||
|
||||
def migrate_repository(
|
||||
client: GiteaClient,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
org_labels: dict[str, dict[str, Any]],
|
||||
issue_mode: str,
|
||||
apply: bool,
|
||||
delete_repo_labels: bool,
|
||||
) -> RepoResult:
|
||||
result = RepoResult()
|
||||
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||
local_labels = {
|
||||
str(label.get("name") or ""): label
|
||||
for label in repo_labels
|
||||
if str(label.get("name") or "") in org_labels
|
||||
}
|
||||
result.matched_local_labels = len(local_labels)
|
||||
if not local_labels:
|
||||
return result
|
||||
|
||||
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||
|
||||
for issue_type in ("issues", "pulls"):
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "type": issue_type},
|
||||
limit=100,
|
||||
)
|
||||
for issue in issues:
|
||||
issue_number = int(issue.get("number") or issue.get("index"))
|
||||
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||
final_label_ids = list(issue_label_ids)
|
||||
changed = False
|
||||
migrated_count = 0
|
||||
local_ids_to_remove: list[int] = []
|
||||
org_ids_to_add: list[int] = []
|
||||
for label in issue.get("labels") or []:
|
||||
local_id = _label_id(label)
|
||||
if local_id is None or local_id not in local_by_id:
|
||||
continue
|
||||
name = str(label.get("name") or "")
|
||||
org_id = org_by_name.get(name)
|
||||
if org_id is None:
|
||||
continue
|
||||
local_ids_to_remove.append(local_id)
|
||||
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||
org_ids_to_add.append(org_id)
|
||||
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||
if org_id not in final_label_ids:
|
||||
final_label_ids.append(org_id)
|
||||
changed = True
|
||||
migrated_count += 1
|
||||
if changed:
|
||||
if apply:
|
||||
if issue_mode == "delete-add":
|
||||
for local_id in local_ids_to_remove:
|
||||
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||
if org_ids_to_add:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": org_ids_to_add},
|
||||
)
|
||||
else:
|
||||
client.request_json(
|
||||
"PUT",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": final_label_ids},
|
||||
)
|
||||
result.issue_label_migrations += migrated_count
|
||||
|
||||
if delete_repo_labels:
|
||||
for name, label in sorted(local_labels.items()):
|
||||
label_id = _label_id(label)
|
||||
if label_id is None:
|
||||
result.errors.append(f"{name} has no label id")
|
||||
continue
|
||||
if apply:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
result.errors.append(f"delete repository label {name}: {exc}")
|
||||
continue
|
||||
result.repo_label_deletions += 1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||
if explicit:
|
||||
return sorted(set(explicit))
|
||||
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||
return [name for name in names if repo_filter.search(name)]
|
||||
|
||||
|
||||
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||
|
||||
|
||||
def _label_id(label: dict[str, Any]) -> int | None:
|
||||
value = label.get("id") or label.get("index")
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize issue labels to a Gitea repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
load_json,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument(
|
||||
"--scope",
|
||||
choices=("repository", "organization"),
|
||||
default="repository",
|
||||
help="sync repository labels or organization labels",
|
||||
)
|
||||
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
labels = _load_labels(args.labels_file)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
org_name = args.org or target.owner
|
||||
|
||||
if args.scope == "organization":
|
||||
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Labels file: {args.labels_file}")
|
||||
|
||||
if not args.apply and not token:
|
||||
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
||||
for label in labels:
|
||||
print(f"would ensure {label['name']} ({label['color']})")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
if args.scope == "organization":
|
||||
existing = _org_labels_by_name(client, org_name)
|
||||
else:
|
||||
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||
creates: list[dict[str, Any]] = []
|
||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
|
||||
for label in labels:
|
||||
current = existing.get(label["name"])
|
||||
if current is None:
|
||||
creates.append(label)
|
||||
continue
|
||||
|
||||
patch = _diff_label(current, label)
|
||||
if patch:
|
||||
updates.append((current, patch))
|
||||
|
||||
if not creates and not updates:
|
||||
print("No label changes needed.")
|
||||
return 0
|
||||
|
||||
for label in creates:
|
||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||
if args.apply:
|
||||
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||
|
||||
for current, patch in updates:
|
||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||
if args.apply:
|
||||
label_id = current.get("id") or current.get("index")
|
||||
if label_id is None:
|
||||
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||
body=patch,
|
||||
)
|
||||
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||
payload = load_json(path)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"{path} must contain a JSON list")
|
||||
|
||||
labels: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(payload, start=1):
|
||||
if not isinstance(item, dict):
|
||||
raise GiteaError(f"label #{index} must be an object")
|
||||
label = {
|
||||
"name": str(item.get("name", "")).strip(),
|
||||
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
||||
"description": str(item.get("description", "")).strip(),
|
||||
"exclusive": bool(item.get("exclusive", False)),
|
||||
}
|
||||
if not label["name"]:
|
||||
raise GiteaError(f"label #{index} is missing name")
|
||||
if label["name"] in seen:
|
||||
raise GiteaError(f"duplicate label name: {label['name']}")
|
||||
if not _is_hex_color(label["color"]):
|
||||
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
||||
seen.add(label["name"])
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(org_path(owner, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, "/labels")
|
||||
return repo_path(owner, repo, "/labels")
|
||||
|
||||
|
||||
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, f"/labels/{label_id}")
|
||||
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||
|
||||
|
||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||
patch: dict[str, Any] = {}
|
||||
if _normalize_color(current.get("color")) != desired["color"]:
|
||||
patch["color"] = desired["color"]
|
||||
if str(current.get("description") or "").strip() != desired["description"]:
|
||||
patch["description"] = desired["description"]
|
||||
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
||||
patch["exclusive"] = desired["exclusive"]
|
||||
return patch
|
||||
|
||||
|
||||
def _normalize_color(value: Any) -> str:
|
||||
return str(value or "").strip().lstrip("#").lower()
|
||||
|
||||
|
||||
def _is_hex_color(value: str) -> bool:
|
||||
if len(value) != 6:
|
||||
return False
|
||||
return all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,637 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror project documentation files into Gitea repository wikis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
||||
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
||||
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
".gitea",
|
||||
".venv",
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"build",
|
||||
".cache",
|
||||
".module-test-build",
|
||||
}
|
||||
EXCLUDED_NAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"uv.lock",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class WikiSource:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
origin: str
|
||||
page_title: str
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
||||
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
||||
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
if args.repo:
|
||||
requested = set(args.repo)
|
||||
repos = {key: repo for key, repo in repos.items() if key in requested}
|
||||
missing = requested - set(repos)
|
||||
if missing:
|
||||
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
||||
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
if args.page:
|
||||
requested_pages = set(args.page)
|
||||
sources = [source for source in sources if source.page_title in requested_pages]
|
||||
missing_pages = requested_pages - {source.page_title for source in sources}
|
||||
if missing_pages:
|
||||
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Wiki source files: {len(sources)}")
|
||||
for repo_key, count in grouped_counts(sources).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
if not args.apply:
|
||||
preview_sources(sources)
|
||||
print("Dry run only. Re-run with --apply to write wiki pages.")
|
||||
return 0
|
||||
|
||||
token = require_token() if args.transport == "api" else ""
|
||||
failures = 0
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
repo = repos[repo_key]
|
||||
try:
|
||||
if args.transport == "git":
|
||||
sync_repo_wiki_git(
|
||||
repo,
|
||||
repo_sources,
|
||||
cache_dir=args.wiki_cache_dir,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
commit_message=args.commit_message,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
else:
|
||||
sync_repo_wiki(
|
||||
repo,
|
||||
repo_sources,
|
||||
token,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
except GiteaError as exc:
|
||||
failures += 1
|
||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
if "git.add-ideas.de" not in result.stdout:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
||||
sources: list[WikiSource] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if is_repo_doc(repo.root, path):
|
||||
rel = path.relative_to(repo.root)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo.key,
|
||||
path=path,
|
||||
origin="repository",
|
||||
page_title=title_for_path("Repo", rel),
|
||||
)
|
||||
)
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if is_product_doc(path):
|
||||
rel = path.relative_to(product_dir)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo_key,
|
||||
path=path,
|
||||
origin=f"product:{product_dir.name}",
|
||||
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
||||
)
|
||||
)
|
||||
|
||||
unique: dict[tuple[str, str], WikiSource] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.page_title)] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
||||
|
||||
|
||||
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
parts = set(path.relative_to(repo_root_path).parts)
|
||||
if parts & EXCLUDED_PARTS:
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
rel = path.relative_to(repo_root_path)
|
||||
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
||||
return True
|
||||
if "generated" in rel.parts:
|
||||
return False
|
||||
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
||||
return True
|
||||
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_product_doc(path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
||||
return False
|
||||
if re.search(
|
||||
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
||||
path.name,
|
||||
re.IGNORECASE,
|
||||
) and path.suffix.lower() in {".md", ".txt"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_file(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in TEXT_SUFFIXES:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
return b"\x00" not in chunk
|
||||
|
||||
|
||||
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
||||
stem = rel.with_suffix("").as_posix()
|
||||
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
||||
return f"{prefix}-{stem}"[:180]
|
||||
|
||||
|
||||
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
||||
grouped: dict[str, list[WikiSource]] = {}
|
||||
for source in sources:
|
||||
grouped.setdefault(source.repo_key, []).append(source)
|
||||
return dict(sorted(grouped.items()))
|
||||
|
||||
|
||||
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
||||
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
||||
|
||||
|
||||
def preview_sources(sources: list[WikiSource]) -> None:
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
print(f"{repo_key}:")
|
||||
for source in repo_sources[:80]:
|
||||
print(f" {source.page_title} <- {source.path}")
|
||||
if len(repo_sources) > 80:
|
||||
print(f" ... {len(repo_sources) - 80} more")
|
||||
|
||||
|
||||
def sync_repo_wiki(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
token: str,
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||
existing_page_names = {
|
||||
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
||||
for page in existing_pages
|
||||
if page.get("title")
|
||||
}
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
changed = put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
source.page_title,
|
||||
content,
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||
|
||||
|
||||
def sync_repo_wiki_git(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
*,
|
||||
cache_dir: pathlib.Path,
|
||||
overwrite_unmanaged: bool,
|
||||
commit_message: str,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
status = write_wiki_page_file(
|
||||
wiki_root,
|
||||
source.page_title,
|
||||
content,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
write_wiki_page_file(
|
||||
wiki_root,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||
if not git_has_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
run_git(wiki_root, "add", "-A")
|
||||
if not git_has_staged_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
ensure_git_identity(wiki_root)
|
||||
run_git(wiki_root, "commit", "-m", commit_message)
|
||||
run_git(wiki_root, "push")
|
||||
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
||||
|
||||
|
||||
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
||||
remote = wiki_remote_for_repo(repo.root)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
||||
if (checkout / ".git").exists():
|
||||
run_git(checkout, "remote", "set-url", "origin", remote)
|
||||
run_git(checkout, "fetch", "origin")
|
||||
branch = current_branch(checkout)
|
||||
if branch:
|
||||
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
||||
else:
|
||||
run_git(checkout, "reset", "--hard")
|
||||
run_git(checkout, "clean", "-fd")
|
||||
return checkout
|
||||
if checkout.exists():
|
||||
shutil.rmtree(checkout)
|
||||
result = subprocess.run(
|
||||
["git", "clone", remote, str(checkout)],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return checkout
|
||||
checkout.mkdir(parents=True, exist_ok=True)
|
||||
run_git(checkout, "init")
|
||||
run_git(checkout, "remote", "add", "origin", remote)
|
||||
run_git(checkout, "checkout", "-b", "master")
|
||||
return checkout
|
||||
|
||||
|
||||
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
||||
remote = result.stdout.strip()
|
||||
if remote.endswith(".git"):
|
||||
return f"{remote[:-4]}.wiki.git"
|
||||
return f"{remote}.wiki.git"
|
||||
|
||||
|
||||
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
||||
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
||||
if path.exists():
|
||||
current = read_text(path)
|
||||
if current == content:
|
||||
return "unchanged"
|
||||
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
||||
return "skipped unmanaged"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return "updated"
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||
for path in sorted(wiki_root.glob("*.md")):
|
||||
if path.name in desired:
|
||||
continue
|
||||
current = read_text(path)
|
||||
if MANAGED_MARKER not in current:
|
||||
continue
|
||||
path.unlink()
|
||||
print(f"deleted stale managed wiki page {path.name}")
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_api(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
existing_page_names: dict[str, str],
|
||||
sources: list[WikiSource],
|
||||
) -> None:
|
||||
desired = {source.page_title for source in sources}
|
||||
desired.add("Codex-Project-Index")
|
||||
for title, page_name in sorted(existing_page_names.items()):
|
||||
if title in desired:
|
||||
continue
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||
continue
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||
|
||||
|
||||
def wiki_file_stem(title: str) -> str:
|
||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||
|
||||
|
||||
def git_has_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "status", "--porcelain"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return False
|
||||
if result.returncode == 1:
|
||||
return True
|
||||
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
||||
|
||||
|
||||
def current_branch(root: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
branch = result.stdout.strip()
|
||||
if result.returncode != 0 or branch == "HEAD":
|
||||
return ""
|
||||
return branch
|
||||
|
||||
|
||||
def ensure_git_identity(root: pathlib.Path) -> None:
|
||||
if not git_config_value(root, "user.email"):
|
||||
run_git(root, "config", "user.email", "codex@govoplan.local")
|
||||
if not git_config_value(root, "user.name"):
|
||||
run_git(root, "config", "user.name", "Codex")
|
||||
|
||||
|
||||
def git_config_value(root: pathlib.Path, key: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "config", "--get", key],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def run_git(root: pathlib.Path, *args: str) -> None:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
command = "git -C " + str(root) + " " + " ".join(args)
|
||||
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
||||
|
||||
|
||||
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
||||
return []
|
||||
raise
|
||||
|
||||
|
||||
def render_wiki_content(source: WikiSource) -> str:
|
||||
raw = read_text(source.path)
|
||||
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||
header = [
|
||||
f"{MANAGED_MARKER}{fingerprint} -->",
|
||||
"",
|
||||
f"> Mirrored from `{source.path}`.",
|
||||
f"> Origin: `{source.origin}`.",
|
||||
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
if source.path.suffix.lower() == ".csv":
|
||||
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
||||
return "\n".join(header) + raw.rstrip() + "\n"
|
||||
|
||||
|
||||
def put_wiki_page(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
title: str,
|
||||
content: str,
|
||||
existing_page_names: dict[str, str],
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
) -> bool:
|
||||
body = {
|
||||
"title": title,
|
||||
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
||||
"message": f"Sync {title} from project files",
|
||||
}
|
||||
if title in existing_page_names:
|
||||
page_name = existing_page_names[title]
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
current_content = decode_wiki_content(current)
|
||||
if current_content == content:
|
||||
return False
|
||||
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
||||
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
||||
return False
|
||||
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
||||
return True
|
||||
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
||||
existing_page_names[title] = title
|
||||
return True
|
||||
|
||||
|
||||
def quote_wiki_page_name(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="+")
|
||||
|
||||
|
||||
def decode_wiki_content(page: dict[str, Any]) -> str:
|
||||
encoded = str(page.get("content_base64") or "")
|
||||
if not encoded:
|
||||
return ""
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return path.read_text(encoding="latin-1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,377 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preview or import inline TODO-style markers as Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
MARKER_RE = re.compile(
|
||||
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
||||
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
||||
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
||||
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DEFAULT_EXCLUDES = (
|
||||
"!**/.git/**",
|
||||
"!**/.venv/**",
|
||||
"!**/node_modules/**",
|
||||
"!**/__pycache__/**",
|
||||
"!**/.module-test-build/**",
|
||||
"!**/dist/**",
|
||||
"!**/build/**",
|
||||
"!**/.cache/**",
|
||||
"!.gitea/**",
|
||||
"!docs/GITEA_ISSUES.md",
|
||||
"!scripts/gitea-todo-import.py",
|
||||
"!scripts/gitea-sync-labels.py",
|
||||
"!scripts/gitea-codex-note.py",
|
||||
"!scripts/gitea_common.py",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
||||
parser.add_argument(
|
||||
"--module-label",
|
||||
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
||||
)
|
||||
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
||||
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
||||
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
markers = scan_markers(root, include_linked=args.include_linked)
|
||||
if args.limit > 0:
|
||||
markers = markers[: args.limit]
|
||||
|
||||
module_label = args.module_label or infer_module_label(root)
|
||||
previews = [
|
||||
build_issue_preview(
|
||||
target.owner,
|
||||
target.repo,
|
||||
marker,
|
||||
module_label=module_label,
|
||||
infer_areas=not args.no_area_labels,
|
||||
extra_labels=args.extra_label,
|
||||
)
|
||||
for marker in markers
|
||||
]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Scanned root: {root}")
|
||||
print(f"Found {len(previews)} importable marker(s).")
|
||||
for preview in previews:
|
||||
print(f"- {preview.title}")
|
||||
print(f" {preview.location}")
|
||||
print(f" labels: {', '.join(preview.labels)}")
|
||||
|
||||
if not args.apply:
|
||||
if not previews:
|
||||
return 0
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
||||
return 0
|
||||
|
||||
if not previews:
|
||||
print("No issues to create.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = load_label_ids(client, target.owner, target.repo)
|
||||
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
||||
if missing_labels:
|
||||
joined = ", ".join(missing_labels)
|
||||
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
||||
|
||||
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
||||
created = 0
|
||||
skipped = 0
|
||||
for preview in previews:
|
||||
if preview.fingerprint in existing_fingerprints:
|
||||
print(f"skip existing {preview.location}")
|
||||
skipped += 1
|
||||
continue
|
||||
issue = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/issues"),
|
||||
body={
|
||||
"title": preview.title,
|
||||
"body": preview.body,
|
||||
"labels": [label_ids[label] for label in preview.labels],
|
||||
},
|
||||
)
|
||||
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
||||
created += 1
|
||||
|
||||
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
||||
self.path = path
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.marker = marker.upper()
|
||||
self.context = context
|
||||
self.text = text
|
||||
self.raw = raw.rstrip("\n")
|
||||
|
||||
@property
|
||||
def location(self) -> str:
|
||||
return f"{self.path}:{self.line}"
|
||||
|
||||
|
||||
class IssuePreview:
|
||||
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
||||
self.title = title
|
||||
self.body = body
|
||||
self.labels = labels
|
||||
self.fingerprint = fingerprint
|
||||
self.location = location
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"title": self.title,
|
||||
"body": self.body,
|
||||
"labels": self.labels,
|
||||
"fingerprint": self.fingerprint,
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
|
||||
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
||||
command = [
|
||||
"rg",
|
||||
"--json",
|
||||
"--hidden",
|
||||
"--line-number",
|
||||
"--column",
|
||||
"-e",
|
||||
r"\b(TODO|FIXME|XXX|HACK)\b",
|
||||
]
|
||||
for pattern in DEFAULT_EXCLUDES:
|
||||
command.extend(["--glob", pattern])
|
||||
command.append(str(root))
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode not in {0, 1}:
|
||||
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
||||
|
||||
markers: list[Marker] = []
|
||||
for line in result.stdout.splitlines():
|
||||
event = json.loads(line)
|
||||
if event.get("type") != "match":
|
||||
continue
|
||||
data = event["data"]
|
||||
raw_line = data["lines"]["text"]
|
||||
match = MARKER_RE.search(raw_line)
|
||||
if not match:
|
||||
continue
|
||||
if not _looks_like_marker(match):
|
||||
continue
|
||||
context = (match.group("context") or "").strip()
|
||||
if not include_linked and _already_linked(context, raw_line):
|
||||
continue
|
||||
markers.append(
|
||||
Marker(
|
||||
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
||||
line=int(data["line_number"]),
|
||||
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
||||
marker=match.group("marker"),
|
||||
context=context,
|
||||
text=clean_marker_text(match.group("text") or raw_line),
|
||||
raw=raw_line,
|
||||
)
|
||||
)
|
||||
return markers
|
||||
|
||||
|
||||
def build_issue_preview(
|
||||
owner: str,
|
||||
repo: str,
|
||||
marker: Marker,
|
||||
*,
|
||||
module_label: str,
|
||||
infer_areas: bool,
|
||||
extra_labels: list[str],
|
||||
) -> IssuePreview:
|
||||
fingerprint = marker_fingerprint(owner, repo, marker)
|
||||
summary = marker.text or f"review {marker.location}"
|
||||
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
||||
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from an inline marker.",
|
||||
"",
|
||||
f"- Repository: `{owner}/{repo}`",
|
||||
f"- Location: `{marker.location}`",
|
||||
f"- Marker: `{marker.marker}`",
|
||||
f"- Context: `{marker.context or 'none'}`",
|
||||
"",
|
||||
"Current line:",
|
||||
"",
|
||||
"```text",
|
||||
marker.raw,
|
||||
"```",
|
||||
"",
|
||||
"Migration rule:",
|
||||
"",
|
||||
"- Keep this Gitea issue as the canonical backlog item.",
|
||||
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
||||
]
|
||||
)
|
||||
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
||||
|
||||
|
||||
def clean_marker_text(text: str) -> str:
|
||||
cleaned = text.strip()
|
||||
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
||||
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
||||
return cleaned
|
||||
|
||||
|
||||
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
||||
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
||||
area = infer_area_label(marker.path) if infer_areas else ""
|
||||
if area:
|
||||
labels.append(area)
|
||||
return labels
|
||||
|
||||
|
||||
def marker_type_label(marker: str) -> str:
|
||||
if marker == "FIXME":
|
||||
return "type/bug"
|
||||
if marker in {"HACK", "XXX"}:
|
||||
return "type/debt"
|
||||
return "type/task"
|
||||
|
||||
|
||||
def infer_area_label(path: pathlib.Path) -> str:
|
||||
text = path.as_posix()
|
||||
if text.startswith("webui/"):
|
||||
return "area/webui"
|
||||
if text.startswith("alembic/"):
|
||||
return "area/migrations"
|
||||
if text.startswith("docs/"):
|
||||
return "area/docs"
|
||||
if text.startswith("scripts/"):
|
||||
return "area/devex"
|
||||
if "/rbac" in text or "permission" in text:
|
||||
return "area/rbac"
|
||||
if "/access" in text or "auth" in text or "session" in text:
|
||||
return "area/auth"
|
||||
if "tenant" in text:
|
||||
return "area/tenancy"
|
||||
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
||||
return "area/governance"
|
||||
if "module" in text:
|
||||
return "area/module-system"
|
||||
if "migration" in text:
|
||||
return "area/migrations"
|
||||
if "router" in text or "api" in text:
|
||||
return "area/api"
|
||||
if "db" in text or "model" in text or "persistence" in text:
|
||||
return "area/db"
|
||||
return ""
|
||||
|
||||
|
||||
def infer_module_label(root: pathlib.Path) -> str:
|
||||
known = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
if root.name in known:
|
||||
return known[root.name]
|
||||
if root.name.startswith("govoplan-"):
|
||||
suffix = root.name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
return "module/core"
|
||||
|
||||
|
||||
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
||||
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def truncate_title(title: str, limit: int = 180) -> str:
|
||||
if len(title) <= limit:
|
||||
return title
|
||||
return f"{title[: limit - 1].rstrip()}..."
|
||||
|
||||
|
||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||
return label_ids_by_name(client, owner, repo)
|
||||
|
||||
|
||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "labels": "source/todo-scan"},
|
||||
)
|
||||
fingerprints: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
||||
fingerprints.add(match.group(1))
|
||||
return fingerprints
|
||||
|
||||
|
||||
def _already_linked(context: str, raw_line: str) -> bool:
|
||||
linked_text = f"{context} {raw_line}".lower()
|
||||
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
||||
|
||||
|
||||
def _looks_like_marker(match: re.Match[str]) -> bool:
|
||||
text = (match.group("text") or "").strip()
|
||||
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for Gitea maintenance scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
class GiteaError(RuntimeError):
|
||||
"""Raised when a Gitea API request fails."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoTarget:
|
||||
base_url: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
||||
|
||||
@property
|
||||
def api_base(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/api/v1"
|
||||
|
||||
|
||||
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return start.resolve()
|
||||
return pathlib.Path(result.stdout.strip()).resolve()
|
||||
|
||||
|
||||
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
||||
env_url = os.environ.get("GITEA_URL")
|
||||
env_owner = os.environ.get("GITEA_OWNER")
|
||||
env_repo = os.environ.get("GITEA_REPO")
|
||||
|
||||
remote_url = _git_remote(root, remote_name)
|
||||
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
||||
|
||||
base_url = (env_url or inferred_url).rstrip("/")
|
||||
owner = env_owner or inferred_owner
|
||||
repo = env_repo or inferred_repo
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name, value in (
|
||||
("GITEA_URL", base_url),
|
||||
("GITEA_OWNER", owner),
|
||||
("GITEA_REPO", repo),
|
||||
)
|
||||
if not value
|
||||
]
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(
|
||||
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
||||
)
|
||||
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
||||
|
||||
|
||||
def quote_path(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="")
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
||||
self.target = target
|
||||
self.token = token
|
||||
|
||||
def request_json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, Any] | None = None,
|
||||
query: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{self.target.api_base}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
||||
|
||||
data = None
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "codex-gitea-maintenance",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
payload = response.read().decode("utf-8")
|
||||
if not payload:
|
||||
return None
|
||||
return json.loads(payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
message = exc.read().decode("utf-8", errors="replace")
|
||||
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
query: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
effective_limit = min(limit, 50)
|
||||
seen_pages: set[tuple[Any, ...]] = set()
|
||||
while True:
|
||||
page_query = dict(query or {})
|
||||
page_query.update({"page": page, "limit": effective_limit})
|
||||
payload = self.request_json("GET", path, query=page_query)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
||||
if not payload:
|
||||
return items
|
||||
signature = tuple(
|
||||
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
||||
for item in payload
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
if signature in seen_pages:
|
||||
return items
|
||||
seen_pages.add(signature)
|
||||
items.extend(payload)
|
||||
if len(payload) < effective_limit:
|
||||
return items
|
||||
page += 1
|
||||
|
||||
|
||||
def load_json(path: pathlib.Path) -> Any:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
|
||||
loaded: list[str] = []
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[len("export ") :].strip()
|
||||
if "=" not in stripped:
|
||||
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = _strip_env_value(value.strip())
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||||
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
||||
if key not in os.environ:
|
||||
os.environ[key] = value
|
||||
loaded.append(key)
|
||||
return loaded
|
||||
|
||||
|
||||
def require_token() -> str:
|
||||
token = os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
raise GiteaError(
|
||||
"GITEA_TOKEN is required when using --apply. "
|
||||
"Set it in the environment, the target .env, or --env-file."
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||
|
||||
|
||||
def org_path(owner: str, suffix: str) -> str:
|
||||
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||
|
||||
|
||||
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Return org labels plus optional repository labels, keyed by label name.
|
||||
|
||||
Gitea stores organization labels separately from repository labels. Issue
|
||||
APIs can use label ids from both scopes on supported instances, so issue
|
||||
creation helpers should resolve both. Repository labels intentionally win
|
||||
when a repo carries a local label with the same name.
|
||||
"""
|
||||
|
||||
labels: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
for label in client.paginate(org_path(owner, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
except GiteaError:
|
||||
# User-owned repositories or older instances may not expose org labels.
|
||||
pass
|
||||
|
||||
if repo:
|
||||
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
|
||||
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||
ids: dict[str, int] = {}
|
||||
for name, label in labels_by_name(client, owner, repo).items():
|
||||
label_id = label.get("id") or label.get("index")
|
||||
if label_id is None:
|
||||
continue
|
||||
ids[name] = int(label_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
||||
if not remote_url:
|
||||
return "", "", ""
|
||||
|
||||
parsed = urllib.parse.urlparse(remote_url)
|
||||
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
||||
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
prefix = "/".join(path_parts[:-2])
|
||||
base_path = f"/{prefix}" if prefix else ""
|
||||
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
||||
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
||||
|
||||
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
||||
if scp_like:
|
||||
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
return f"https://{scp_like.group('host')}", owner, repo
|
||||
|
||||
return "", "", ""
|
||||
|
||||
|
||||
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
||||
if len(path_parts) < 2:
|
||||
return "", ""
|
||||
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
||||
|
||||
|
||||
def _strip_git_suffix(value: str) -> str:
|
||||
return value[:-4] if value.endswith(".git") else value
|
||||
|
||||
|
||||
def _strip_env_value(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
return value[1:-1]
|
||||
return value
|
||||
@@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WEBUI_DIR="${1:-$ROOT/webui}"
|
||||
WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-webui-release-deps.XXXXXXXX")"
|
||||
GOVOPLAN_DEPS="$WORK_ROOT/govoplan-webui-deps.tsv"
|
||||
export GOVOPLAN_DEPS
|
||||
|
||||
trap 'rm -rf "$WORK_ROOT"' EXIT
|
||||
|
||||
retry() {
|
||||
local attempt status
|
||||
for attempt in 1 2 3; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
status=$?
|
||||
if [[ "$attempt" == 3 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
sleep $((attempt * 10))
|
||||
done
|
||||
}
|
||||
|
||||
cd "$WEBUI_DIR"
|
||||
|
||||
node <<'JS'
|
||||
const fs = require("fs");
|
||||
|
||||
const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
||||
const rel = JSON.parse(fs.readFileSync("package.release.json", "utf8"));
|
||||
const govoplanDeps = [];
|
||||
const dependencies = {...(rel.dependencies || {})};
|
||||
|
||||
for (const [name, spec] of Object.entries(dependencies)) {
|
||||
if (name.startsWith("@govoplan/")) {
|
||||
govoplanDeps.push([name, spec]);
|
||||
delete dependencies[name];
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of ["devDependencies", "peerDependencies", "optionalDependencies", "overrides"]) {
|
||||
if (rel[key]) pkg[key] = rel[key];
|
||||
}
|
||||
pkg.dependencies = dependencies;
|
||||
|
||||
fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
|
||||
fs.writeFileSync(process.env.GOVOPLAN_DEPS, govoplanDeps.map((item) => item.join("\t")).join("\n") + "\n");
|
||||
JS
|
||||
|
||||
rm -f package-lock.json
|
||||
npm cache clean --force
|
||||
retry npm install --prefer-online
|
||||
|
||||
module_paths=()
|
||||
while IFS=$'\t' read -r package_name spec; do
|
||||
[[ -n "${package_name:-}" ]] || continue
|
||||
git_url="${spec%%#*}"
|
||||
git_ref="${spec#*#}"
|
||||
if [[ "$git_url" == "$spec" || -z "$git_ref" ]]; then
|
||||
echo "Unsupported GovOPlaN WebUI git spec for $package_name: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
clone_url="${git_url#git+}"
|
||||
clone_dir="$WORK_ROOT/${package_name#@govoplan/}"
|
||||
echo "Installing $package_name from $clone_url#$git_ref"
|
||||
retry git clone --depth 1 --branch "$git_ref" "$clone_url" "$clone_dir"
|
||||
module_paths+=("file:$clone_dir")
|
||||
done < "$GOVOPLAN_DEPS"
|
||||
|
||||
if [[ "${#module_paths[@]}" -gt 0 ]]; then
|
||||
retry npm install --prefer-online --no-save --install-links "${module_paths[@]}"
|
||||
fi
|
||||
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||
WEBUI_ROOT="$ROOT/webui"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||
NPM="${NPM:-$NODE_BIN/npm}"
|
||||
|
||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||
|
||||
backend_pid=""
|
||||
frontend_pid=""
|
||||
|
||||
fail() {
|
||||
printf 'launch-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
case "$DEV_DATABASE_BACKEND" in
|
||||
postgres)
|
||||
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
;;
|
||||
sqlite)
|
||||
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
||||
;;
|
||||
*)
|
||||
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
||||
;;
|
||||
esac
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind((host, port))
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
"$PYTHON" - "$1" <<'PY'
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
deadline = time.monotonic() + 60
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if 200 <= response.status < 500:
|
||||
raise SystemExit(0)
|
||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||
kill "$frontend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||
kill "$backend_pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
|
||||
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
|
||||
fi
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||
) >"$BACKEND_LOG" 2>&1 &
|
||||
backend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||
wait_for_url "$BACKEND_URL/health" || {
|
||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||
fail "backend did not become healthy"
|
||||
}
|
||||
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
frontend_args=(run dev)
|
||||
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
|
||||
frontend_args+=(-- --force)
|
||||
fi
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \
|
||||
CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" "${frontend_args[@]}"
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||
wait_for_url "$FRONTEND_URL" || {
|
||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||
fail "frontend did not become reachable"
|
||||
}
|
||||
|
||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
GovOPlaN is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
DB: $DATABASE_URL
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Frontend reload:
|
||||
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
|
||||
Vite --force: $FRONTEND_FORCE_RELOAD
|
||||
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
|
||||
|
||||
Press Ctrl+C to stop both processes.
|
||||
EOF
|
||||
|
||||
wait
|
||||
@@ -1,241 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||
if [ ! -f "$PROFILE_ENV" ]; then
|
||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||
fi
|
||||
|
||||
WEBUI_ROOT="$ROOT/webui"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||
NPM="${NPM:-$NODE_BIN/npm}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
|
||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/production-like/logs"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
WORKER_LOG="$LOG_DIR/worker.log"
|
||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||
|
||||
backend_pid=""
|
||||
worker_pid=""
|
||||
frontend_pid=""
|
||||
|
||||
fail() {
|
||||
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$PROFILE_ENV"
|
||||
set +a
|
||||
|
||||
export APP_ENV="${APP_ENV:-staging}"
|
||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
||||
|
||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
||||
from cryptography.fernet import Fernet
|
||||
print(Fernet.generate_key().decode())
|
||||
PY
|
||||
)"
|
||||
export MASTER_KEY_B64
|
||||
fi
|
||||
|
||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||
|
||||
compose() {
|
||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind((host, port))
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_tcp() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=2):
|
||||
raise SystemExit(0)
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
"$PYTHON" - "$1" <<'PY'
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if 200 <= response.status < 500:
|
||||
raise SystemExit(0)
|
||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||
kill "$frontend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||
kill "$backend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
||||
kill "$worker_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
||||
compose down >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||
|
||||
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$WORKER_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
printf 'Starting production-like dependencies through Docker Compose\n'
|
||||
compose up -d
|
||||
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
||||
|
||||
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||
)
|
||||
|
||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||
) >"$BACKEND_LOG" 2>&1 &
|
||||
backend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||
wait_for_url "$BACKEND_URL/health" || {
|
||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||
fail "backend did not become healthy"
|
||||
}
|
||||
|
||||
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
||||
--queues "$CELERY_QUEUES" \
|
||||
--hostname "govoplan-production-like@%h" \
|
||||
--loglevel INFO
|
||||
) >"$WORKER_LOG" 2>&1 &
|
||||
worker_pid="$!"
|
||||
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
||||
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" run dev -- --force
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||
wait_for_url "$FRONTEND_URL" || {
|
||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||
fail "frontend did not become reachable"
|
||||
}
|
||||
|
||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
GovOPlaN production-like development profile is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
Ops: $BACKEND_URL/api/v1/ops/status
|
||||
DB: $DATABASE_URL
|
||||
Redis: $REDIS_URL
|
||||
Files: $FILE_STORAGE_LOCAL_ROOT
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
Worker: $WORKER_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
||||
Press Ctrl+C to stop API, worker, and WebUI.
|
||||
EOF
|
||||
|
||||
wait
|
||||
@@ -1,512 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
os.environ.setdefault("APP_ENV", "test")
|
||||
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
||||
os.environ.setdefault("CELERY_ENABLED", "false")
|
||||
|
||||
from govoplan_core.admin.models import SystemSettings
|
||||
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
||||
from govoplan_core.core.module_installer import (
|
||||
cancel_module_installer_request,
|
||||
claim_next_module_installer_request,
|
||||
module_installer_daemon_status,
|
||||
module_installer_lock_status,
|
||||
queue_module_installer_request,
|
||||
read_module_installer_run,
|
||||
retry_module_installer_request,
|
||||
rollback_module_install_run,
|
||||
run_module_install_plan,
|
||||
supervise_module_install_plan,
|
||||
update_module_installer_daemon_status,
|
||||
update_module_installer_request,
|
||||
)
|
||||
from govoplan_core.core.module_management import (
|
||||
ModuleInstallPlan,
|
||||
ModuleInstallPlanItem,
|
||||
saved_desired_enabled_modules,
|
||||
saved_module_install_plan,
|
||||
save_desired_enabled_modules,
|
||||
save_module_install_plan,
|
||||
)
|
||||
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.server.registry import available_module_manifests
|
||||
|
||||
|
||||
Scenario = Callable[[Path], dict[str, object]]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
||||
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
||||
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
||||
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
||||
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
||||
args = parser.parse_args()
|
||||
|
||||
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
||||
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||
selected = args.scenario or sorted(SCENARIOS)
|
||||
results: list[dict[str, object]] = []
|
||||
ok = True
|
||||
try:
|
||||
for name in selected:
|
||||
scenario_root = runtime_root / name
|
||||
scenario_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
result = SCENARIOS[name](scenario_root)
|
||||
except Exception as exc:
|
||||
ok = False
|
||||
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
||||
else:
|
||||
ok = ok and bool(result.get("ok"))
|
||||
results.append(result)
|
||||
finally:
|
||||
if not args.keep_runtime and args.runtime_root is None:
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
|
||||
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
||||
if args.format == "json":
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
for result in results:
|
||||
status = "ok" if result.get("ok") else "FAILED"
|
||||
print(f"{status}: {result['scenario']}")
|
||||
if result.get("run_id"):
|
||||
print(f" run: {result['run_id']}")
|
||||
if result.get("record_path"):
|
||||
print(f" record: {result['record_path']}")
|
||||
if result.get("error"):
|
||||
print(f" error: {result['error']}")
|
||||
print(f"runtime: {runtime_root}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def package_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "package-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_package_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
restored_plan = saved_module_install_plan(session)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"package-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def migration_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "migration-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"migration-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def restart_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "restart-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-fails",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"restart-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
||||
)
|
||||
|
||||
|
||||
def health_timeout(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "health-timeout-example")
|
||||
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-ok",
|
||||
health_url="http://127.0.0.1:9/health",
|
||||
health_timeout_seconds=0.2,
|
||||
health_interval_seconds=0.05,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"health-timeout",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
available = {"retirement-example": _retirement_failure_manifest()}
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "retirement-example",
|
||||
"action": "uninstall",
|
||||
"python_package": "govoplan-retirement-example",
|
||||
"destroy_data": True,
|
||||
}])
|
||||
session.commit()
|
||||
with _patch_subprocess(_success_run):
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"destructive-retirement-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def postgres_hooks(root: Path) -> dict[str, object]:
|
||||
_, database = _sqlite_database(root)
|
||||
backup_script = (
|
||||
"import json, os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
||||
)
|
||||
check_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
||||
)
|
||||
restore_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
||||
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
||||
)
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "postgres-hook-example",
|
||||
"action": "install",
|
||||
"python_package": "govoplan-postgres-hook-example",
|
||||
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
||||
}])
|
||||
session.commit()
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
||||
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
||||
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
||||
dry_run=True,
|
||||
)
|
||||
with _patch_subprocess(_non_shell_success_run):
|
||||
rollback = rollback_module_install_run(
|
||||
run_id=result.run_id,
|
||||
runtime_dir=root / "installer",
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
)
|
||||
run_dir = result.record_path.parent
|
||||
record = _run_record(root, result.run_id)
|
||||
ok = (
|
||||
result.status == "dry-run"
|
||||
and rollback.status == "rolled-back"
|
||||
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
||||
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
||||
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
||||
)
|
||||
return {
|
||||
"scenario": "postgres-hooks",
|
||||
"ok": ok,
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"rollback_status": rollback.status,
|
||||
}
|
||||
|
||||
|
||||
def queue_and_lock(root: Path) -> dict[str, object]:
|
||||
runtime_dir = root / "installer"
|
||||
status = update_module_installer_daemon_status(
|
||||
runtime_dir=runtime_dir,
|
||||
patch={"status": "polling", "current_request_id": None},
|
||||
)
|
||||
request = queue_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
requested_by="drill",
|
||||
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
||||
)
|
||||
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
||||
assert claimed is not None
|
||||
failed = update_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
request_id=str(claimed["request_id"]),
|
||||
patch={"status": "failed", "error": "simulated drill failure"},
|
||||
)
|
||||
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
||||
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
||||
lock_path = runtime_dir / "install.lock"
|
||||
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
||||
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
||||
lock_path.unlink()
|
||||
return {
|
||||
"scenario": "queue-and-lock",
|
||||
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
||||
"root": str(root),
|
||||
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
||||
}
|
||||
|
||||
|
||||
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
||||
database_url = f"sqlite:///{root / 'drill.db'}"
|
||||
configure_database(database_url)
|
||||
database = get_database()
|
||||
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
||||
return database_url, database
|
||||
|
||||
|
||||
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": module_id,
|
||||
"action": "install",
|
||||
"python_package": f"govoplan-{module_id}",
|
||||
"python_ref": f"govoplan-{module_id}==0.1.0",
|
||||
}])
|
||||
save_desired_enabled_modules(session, ("tenancy", "access"))
|
||||
session.commit()
|
||||
return plan
|
||||
|
||||
|
||||
def _retirement_failure_manifest() -> ModuleManifest:
|
||||
def destroy(_session: object, _module_id: str) -> None:
|
||||
raise RuntimeError("simulated destructive retirement failure")
|
||||
|
||||
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
||||
return MigrationRetirementPlan(
|
||||
supported=True,
|
||||
summary="Drill retirement provider supports destructive cleanup.",
|
||||
destroy_data_supported=True,
|
||||
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
||||
destroy_data_executor=destroy,
|
||||
)
|
||||
|
||||
return ModuleManifest(
|
||||
id="retirement-example",
|
||||
name="Retirement example",
|
||||
version="0.1.0",
|
||||
migration_spec=MigrationSpec(
|
||||
module_id="retirement-example",
|
||||
retirement_supported=True,
|
||||
retirement_provider=provider,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _scenario_result(
|
||||
scenario: str,
|
||||
root: Path,
|
||||
result: Any,
|
||||
*,
|
||||
expected_status: str,
|
||||
checks: dict[str, bool],
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"scenario": scenario,
|
||||
"ok": result.status == expected_status and all(checks.values()),
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"status": result.status,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
||||
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
||||
|
||||
|
||||
def _nested(value: dict[str, object], *keys: str) -> object:
|
||||
current: object = value
|
||||
for key in keys:
|
||||
if not isinstance(current, dict):
|
||||
return None
|
||||
current = current.get(key)
|
||||
return current
|
||||
|
||||
|
||||
def _command_text(argv: object) -> str:
|
||||
if isinstance(argv, list | tuple):
|
||||
return " ".join(str(item) for item in argv)
|
||||
return str(argv)
|
||||
|
||||
|
||||
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
||||
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip freeze" in text:
|
||||
return _completed(stdout="govoplan-core==0.0.0\n")
|
||||
return _completed()
|
||||
|
||||
|
||||
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
||||
if kwargs.get("shell"):
|
||||
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip install" in text and "==" in text and "-r" not in text:
|
||||
return _completed(23, stderr="simulated package install failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "govoplan_core.commands.init_db" in text:
|
||||
return _completed(24, stderr="simulated migration failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
||||
return _completed(25, stderr="simulated restart failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.subprocess.run
|
||||
installer.subprocess.run = fake_run # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.subprocess.run = original # type: ignore[assignment]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_urlopen() -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.urllib.request.urlopen
|
||||
|
||||
def fail(*_args: object, **_kwargs: object) -> None:
|
||||
raise URLError("simulated health timeout")
|
||||
|
||||
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
||||
|
||||
|
||||
SCENARIOS: dict[str, Scenario] = {
|
||||
"destructive-retirement-failure": destructive_retirement_failure,
|
||||
"health-timeout": health_timeout,
|
||||
"migration-failure": migration_failure,
|
||||
"package-failure": package_failure,
|
||||
"postgres-hooks": postgres_hooks,
|
||||
"queue-and-lock": queue_and_lock,
|
||||
"restart-failure": restart_failure,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,128 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
|
||||
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
||||
("core-only", "tenancy,access,admin"),
|
||||
("files-only", "tenancy,access,admin,files"),
|
||||
("mail-only", "tenancy,access,admin,mail"),
|
||||
("campaign-only", "tenancy,access,admin,campaigns"),
|
||||
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
||||
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
||||
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database-url",
|
||||
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
||||
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--module-set",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME=MODULES",
|
||||
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reset-schema",
|
||||
action="store_true",
|
||||
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
||||
)
|
||||
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
||||
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
||||
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
||||
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
||||
args = parser.parse_args()
|
||||
|
||||
database_url = (args.database_url or "").strip()
|
||||
if not database_url:
|
||||
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
||||
if not database_url.startswith("postgresql"):
|
||||
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
||||
|
||||
module_sets = _parse_module_sets(args.module_set)
|
||||
ok = True
|
||||
for name, modules in module_sets:
|
||||
print(f"\n== {name}: {modules} ==")
|
||||
if args.reset_schema:
|
||||
_reset_public_schema(database_url)
|
||||
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
||||
if not args.skip_migrate:
|
||||
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
||||
if not args.skip_smoke:
|
||||
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
||||
if not raw:
|
||||
return DEFAULT_MODULE_SETS
|
||||
parsed: list[tuple[str, str]] = []
|
||||
for item in raw:
|
||||
name, separator, modules = item.partition("=")
|
||||
if not separator or not name.strip() or not modules.strip():
|
||||
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
||||
parsed.append((name.strip(), modules.strip()))
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env["APP_ENV"] = app_env
|
||||
env["DATABASE_URL"] = database_url
|
||||
env["ENABLED_MODULES"] = modules
|
||||
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
||||
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
||||
env["CELERY_ENABLED"] = "false"
|
||||
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
||||
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
||||
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
||||
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
||||
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
||||
return env
|
||||
|
||||
|
||||
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
||||
print("+ " + " ".join(command))
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
||||
if result.returncode != 0:
|
||||
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _reset_public_schema(database_url: str) -> None:
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
||||
|
||||
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
||||
connection.exec_driver_sql("CREATE SCHEMA public")
|
||||
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
||||
finally:
|
||||
engine.dispose()
|
||||
print("Reset PostgreSQL schema: public")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||
if [ ! -f "$PROFILE_ENV" ]; then
|
||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||
fi
|
||||
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/production-like-dev.sh <command> [--yes]
|
||||
|
||||
Commands:
|
||||
start Start the production-like API, worker, WebUI, PostgreSQL, and Redis profile.
|
||||
stop Stop PostgreSQL and Redis profile containers.
|
||||
status Show profile container status and validate the effective environment.
|
||||
seed Run migrations and seed development data against the profile database.
|
||||
reset --yes Stop containers, remove profile volumes, and remove runtime/production-like data.
|
||||
validate-config Validate the profile environment only.
|
||||
|
||||
The start command delegates to scripts/launch-production-like-dev.sh. Stop/reset
|
||||
operate on Docker dependencies; API/WebUI/worker processes started by the
|
||||
launcher are stopped with Ctrl+C in that launcher terminal.
|
||||
EOF
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'production-like-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command="${1:-}"
|
||||
if [ -z "$command" ] || [ "$command" = "-h" ] || [ "$command" = "--help" ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
shift || true
|
||||
|
||||
yes=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--yes) yes=1 ;;
|
||||
*) fail "unknown argument: $arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$PROFILE_ENV"
|
||||
set +a
|
||||
|
||||
export APP_ENV="${APP_ENV:-staging}"
|
||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}"
|
||||
|
||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||
MASTER_KEY_B64="$("$PYTHON" -m govoplan_core.commands.config env-template --profile production-like --generate-secrets | sed -n 's/^MASTER_KEY_B64=//p' | head -n 1)"
|
||||
export MASTER_KEY_B64
|
||||
fi
|
||||
|
||||
compose() {
|
||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
validate_config() {
|
||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||
}
|
||||
|
||||
case "$command" in
|
||||
start)
|
||||
exec "$ROOT/scripts/launch-production-like-dev.sh"
|
||||
;;
|
||||
stop)
|
||||
compose down
|
||||
;;
|
||||
status)
|
||||
compose ps
|
||||
validate_config
|
||||
;;
|
||||
seed)
|
||||
validate_config
|
||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||
;;
|
||||
reset)
|
||||
[ "$yes" = "1" ] || fail "reset is destructive; rerun with --yes"
|
||||
compose down -v
|
||||
rm -rf "$ROOT/runtime/production-like"
|
||||
;;
|
||||
validate-config)
|
||||
validate_config
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
fail "unknown command: $command"
|
||||
;;
|
||||
esac
|
||||
@@ -1,259 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
|
||||
|
||||
Generates a signed module package catalog for a GovOPlaN release, writes it into
|
||||
govoplan-web/public/catalogs/v1, validates it, optionally builds the website,
|
||||
and optionally commits/tags/pushes govoplan-web.
|
||||
|
||||
Options:
|
||||
--version <x.y.z> Release version without leading v.
|
||||
--channel <name> Catalog channel. Defaults to stable.
|
||||
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
||||
--expires-days <days> Catalog expiry window. Defaults to 90.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Ed25519 private key. May be repeated for rotation.
|
||||
--core-root <path> govoplan-core checkout. Defaults to this script parent.
|
||||
--web-root <path> govoplan-web checkout. Defaults to ../govoplan-web.
|
||||
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
|
||||
--npm <path> npm executable used for optional web build.
|
||||
--build-web Run npm run build in govoplan-web after generating files.
|
||||
--commit Commit generated catalog files in govoplan-web.
|
||||
--tag Tag govoplan-web with catalog-v<x.y.z>. Implies --commit.
|
||||
--push Push govoplan-web branch and tag. Implies --commit.
|
||||
--remote <name> Git remote for push. Defaults to origin.
|
||||
--branch <name> Branch to push. Defaults to current govoplan-web branch.
|
||||
-n, --dry-run Print commands and validate inputs without writing git changes.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
CORE_ROOT="$ROOT"
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
VERSION=""
|
||||
CHANNEL="stable"
|
||||
SEQUENCE=""
|
||||
EXPIRES_DAYS="90"
|
||||
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
BUILD_WEB=0
|
||||
COMMIT=0
|
||||
TAG_WEB=0
|
||||
PUSH=0
|
||||
DRY_RUN=0
|
||||
SIGNING_KEYS=()
|
||||
NPM_BIN="${NPM:-}"
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
VERSION="${2#v}"
|
||||
shift 2
|
||||
;;
|
||||
--channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--core-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CORE_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--public-base-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
PUBLIC_BASE_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web)
|
||||
BUILD_WEB=1
|
||||
shift
|
||||
;;
|
||||
--commit)
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--tag)
|
||||
TAG_WEB=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--push)
|
||||
PUSH=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--remote)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--branch)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$VERSION" ]] || fail "--version is required"
|
||||
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
||||
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
||||
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||
fi
|
||||
|
||||
if [[ -z "$BRANCH" ]]; then
|
||||
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
||||
fi
|
||||
[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch"
|
||||
|
||||
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
||||
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
||||
TAG_NAME="catalog-v$VERSION"
|
||||
|
||||
run() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
GEN_ARGS=(
|
||||
"$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py"
|
||||
--version "$VERSION"
|
||||
--channel "$CHANNEL"
|
||||
--expires-days "$EXPIRES_DAYS"
|
||||
--catalog-output "$CATALOG_PATH"
|
||||
--keyring-output "$KEYRING_PATH"
|
||||
--public-base-url "$PUBLIC_BASE_URL"
|
||||
)
|
||||
if [[ -n "$SEQUENCE" ]]; then
|
||||
GEN_ARGS+=(--sequence "$SEQUENCE")
|
||||
fi
|
||||
for signing_key in "${SIGNING_KEYS[@]}"; do
|
||||
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
|
||||
run "${GEN_ARGS[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
|
||||
"$PYTHON" -m govoplan_core.commands.module_installer \
|
||||
--validate-package-catalog "$CATALOG_PATH" \
|
||||
--require-signed-catalog \
|
||||
--approved-catalog-channel "$CHANNEL" \
|
||||
--format json >/dev/null
|
||||
else
|
||||
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
|
||||
fi
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
|
||||
fi
|
||||
|
||||
if [[ "$COMMIT" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$WEB_ROOT" diff --cached --quiet; then
|
||||
echo "No govoplan-web catalog changes to commit."
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
|
||||
fail "govoplan-web tag already exists: $TAG_NAME"
|
||||
fi
|
||||
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
|
||||
if [[ "$PUSH" -eq 1 ]]; then
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
|
||||
else
|
||||
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Catalog ready:"
|
||||
echo " $CATALOG_PATH"
|
||||
echo " $KEYRING_PATH"
|
||||
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"
|
||||
@@ -1,862 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/push-release-tag.sh [options]
|
||||
|
||||
Bumps installable GovOPlaN package versions, commits pending changes in the
|
||||
GovOPlaN release/module repositories, creates an annotated release tag in each
|
||||
repo, and pushes branch+tag.
|
||||
|
||||
By default the script asks which version part to bump:
|
||||
major X.Y.Z -> (X+1).0.0
|
||||
minor X.Y.Z -> X.(Y+1).0
|
||||
subversion X.Y.Z -> X.Y.(Z+1)
|
||||
|
||||
Options:
|
||||
--bump <kind> Version bump: major, minor, subversion, or patch.
|
||||
--version <x.y.z> Explicit target version instead of a bump. May equal
|
||||
the current repo versions when they were bumped already.
|
||||
-r, --remote <name> Remote to push to. Defaults to origin.
|
||||
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
|
||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||
--warn-migration-audit Run the migration release audit without baseline strictness.
|
||||
--strict-migration-audit
|
||||
Require current Alembic heads to be recorded in the
|
||||
latest migration release baseline.
|
||||
--skip-migration-audit Do not run the release migration audit.
|
||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||
catalog into govoplan-web.
|
||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||
Defaults to ../govoplan-web.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Catalog signing key for --publish-web-catalog. May be
|
||||
repeated during key rotation.
|
||||
--catalog-channel <name>
|
||||
Catalog channel. Defaults to stable.
|
||||
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
|
||||
--catalog-expires-days <days>
|
||||
Catalog expiry window. Defaults to 90.
|
||||
--catalog-public-url <url>
|
||||
Public website URL. Defaults to
|
||||
https://govoplan.add-ideas.de.
|
||||
--build-web-catalog Run npm run build in govoplan-web before committing the
|
||||
catalog update.
|
||||
--npm <path> npm executable for lockfile generation.
|
||||
-y, --yes Do not prompt before committing, tagging, and pushing.
|
||||
-n, --dry-run Print intended actions without changing files or git state.
|
||||
-h, --help Show this help.
|
||||
|
||||
Repos:
|
||||
Installable release packages:
|
||||
govoplan-access
|
||||
govoplan-admin
|
||||
govoplan-tenancy
|
||||
govoplan-organizations
|
||||
govoplan-identity
|
||||
govoplan-idm
|
||||
govoplan-policy
|
||||
govoplan-audit
|
||||
govoplan-dashboard
|
||||
govoplan-files
|
||||
govoplan-mail
|
||||
govoplan-campaign
|
||||
govoplan-calendar
|
||||
govoplan-ops
|
||||
govoplan-core
|
||||
|
||||
Roadmap/scaffold module repositories are tag-only until they have package
|
||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
xrechnung, and xta-osci.
|
||||
|
||||
Examples:
|
||||
scripts/push-release-tag.sh
|
||||
scripts/push-release-tag.sh --bump subversion
|
||||
scripts/push-release-tag.sh --version 0.2.0 --message "Release v0.2.0"
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
PACKAGE_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-access"
|
||||
"$PARENT/govoplan-admin"
|
||||
"$PARENT/govoplan-approvals"
|
||||
"$PARENT/govoplan-assets"
|
||||
"$PARENT/govoplan-audit"
|
||||
"$PARENT/govoplan-booking"
|
||||
"$PARENT/govoplan-calendar"
|
||||
"$PARENT/govoplan-campaign"
|
||||
"$PARENT/govoplan-certificates"
|
||||
"$PARENT/govoplan-committee"
|
||||
"$PARENT/govoplan-consultation"
|
||||
"$PARENT/govoplan-contracts"
|
||||
"$PARENT/govoplan-dashboard"
|
||||
"$PARENT/govoplan-docs"
|
||||
"$PARENT/govoplan-facilities"
|
||||
"$PARENT/govoplan-files"
|
||||
"$PARENT/govoplan-forms-runtime"
|
||||
"$PARENT/govoplan-grants"
|
||||
"$PARENT/govoplan-helpdesk"
|
||||
"$PARENT/govoplan-identity"
|
||||
"$PARENT/govoplan-idm"
|
||||
"$PARENT/govoplan-inspections"
|
||||
"$PARENT/govoplan-issue-reporting"
|
||||
"$PARENT/govoplan-learning"
|
||||
"$PARENT/govoplan-mail"
|
||||
"$PARENT/govoplan-ops"
|
||||
"$PARENT/govoplan-organizations"
|
||||
"$PARENT/govoplan-permits"
|
||||
"$PARENT/govoplan-policy"
|
||||
"$PARENT/govoplan-procurement"
|
||||
"$PARENT/govoplan-records"
|
||||
"$PARENT/govoplan-resources"
|
||||
"$PARENT/govoplan-risk-compliance"
|
||||
"$PARENT/govoplan-tenancy"
|
||||
"$PARENT/govoplan-transparency"
|
||||
)
|
||||
TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-addresses"
|
||||
"$PARENT/govoplan-appointments"
|
||||
"$PARENT/govoplan-cases"
|
||||
"$PARENT/govoplan-connectors"
|
||||
"$PARENT/govoplan-dms"
|
||||
"$PARENT/govoplan-erp"
|
||||
"$PARENT/govoplan-fit-connect"
|
||||
"$PARENT/govoplan-forms"
|
||||
"$PARENT/govoplan-identity-trust"
|
||||
"$PARENT/govoplan-ledger"
|
||||
"$PARENT/govoplan-notifications"
|
||||
"$PARENT/govoplan-payments"
|
||||
"$PARENT/govoplan-portal"
|
||||
"$PARENT/govoplan-postbox"
|
||||
"$PARENT/govoplan-reporting"
|
||||
"$PARENT/govoplan-scheduling"
|
||||
"$PARENT/govoplan-search"
|
||||
"$PARENT/govoplan-tasks"
|
||||
"$PARENT/govoplan-templates"
|
||||
"$PARENT/govoplan-workflow"
|
||||
"$PARENT/govoplan-xoev"
|
||||
"$PARENT/govoplan-xrechnung"
|
||||
"$PARENT/govoplan-xta-osci"
|
||||
)
|
||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
||||
REPOS=(
|
||||
"${MODULE_REPOS[@]}"
|
||||
"$ROOT"
|
||||
)
|
||||
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
BUMP=""
|
||||
TARGET_VERSION=""
|
||||
COMMIT_MESSAGE=""
|
||||
TAG_MESSAGE=""
|
||||
DRY_RUN=0
|
||||
YES=0
|
||||
GENERATE_RELEASE_LOCK=1
|
||||
MIGRATION_AUDIT_MODE="auto"
|
||||
NPM_BIN="${NPM:-}"
|
||||
PUBLISH_WEB_CATALOG=0
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
CATALOG_CHANNEL="stable"
|
||||
CATALOG_SEQUENCE=""
|
||||
CATALOG_EXPIRES_DAYS="90"
|
||||
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
|
||||
BUILD_WEB_CATALOG=0
|
||||
CATALOG_SIGNING_KEYS=()
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o BatchMode=yes -o ConnectTimeout=15}"
|
||||
GIT_REMOTE_TIMEOUT="${GIT_REMOTE_TIMEOUT:-30}"
|
||||
|
||||
normalize_bump() {
|
||||
local value="${1,,}"
|
||||
case "$value" in
|
||||
major|minor|subversion)
|
||||
printf '%s\n' "$value"
|
||||
;;
|
||||
patch|sub|sub-version)
|
||||
printf '%s\n' "subversion"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_version() {
|
||||
[[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]
|
||||
}
|
||||
|
||||
version_gt() {
|
||||
local new_major new_minor new_patch old_major old_minor old_patch
|
||||
IFS=. read -r new_major new_minor new_patch <<< "$1"
|
||||
IFS=. read -r old_major old_minor old_patch <<< "$2"
|
||||
|
||||
(( new_major > old_major )) && return 0
|
||||
(( new_major < old_major )) && return 1
|
||||
(( new_minor > old_minor )) && return 0
|
||||
(( new_minor < old_minor )) && return 1
|
||||
(( new_patch > old_patch )) && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
bump_version() {
|
||||
local version="$1"
|
||||
local bump="$2"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
|
||||
case "$bump" in
|
||||
major)
|
||||
printf '%s.0.0\n' "$((major + 1))"
|
||||
;;
|
||||
minor)
|
||||
printf '%s.%s.0\n' "$major" "$((minor + 1))"
|
||||
;;
|
||||
subversion)
|
||||
printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))"
|
||||
;;
|
||||
*)
|
||||
fail "unknown bump kind: $bump"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
get_project_name() {
|
||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["name"])' "$1/pyproject.toml"
|
||||
}
|
||||
|
||||
get_project_version() {
|
||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$1/pyproject.toml"
|
||||
}
|
||||
|
||||
update_pyproject() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
|
||||
"$PYTHON" - "$repo/pyproject.toml" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
text = path.read_text()
|
||||
data = tomllib.loads(text)
|
||||
project = data["project"]
|
||||
name = project["name"]
|
||||
old_version = project["version"]
|
||||
|
||||
text, version_count = re.subn(
|
||||
r'(?m)^version\s*=\s*["\'][^"\']+["\']\s*$',
|
||||
f'version = "{new_version}"',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if version_count != 1:
|
||||
raise SystemExit(f"could not update project.version in {path}")
|
||||
|
||||
if name != "govoplan-core":
|
||||
text, dependency_count = re.subn(
|
||||
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
|
||||
rf'\g<1>{new_version}',
|
||||
text,
|
||||
)
|
||||
if dependency_count < 1:
|
||||
raise SystemExit(f"could not update govoplan-core dependency in {path}")
|
||||
|
||||
path.write_text(text)
|
||||
print(f"{name}: {old_version} -> {new_version}")
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_manifest_version() {
|
||||
local repo="$1"
|
||||
local project_name="$2"
|
||||
local version="$3"
|
||||
local manifest_paths=()
|
||||
|
||||
if [[ "$project_name" == "govoplan-core" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS= read -r -d '' manifest_path; do
|
||||
manifest_paths+=("$manifest_path")
|
||||
done < <(find "$repo/src" -path "*/backend/manifest.py" -print0 2>/dev/null)
|
||||
|
||||
[[ "${#manifest_paths[@]}" -gt 0 ]] || fail "missing module manifest under $repo/src"
|
||||
|
||||
for manifest_path in "${manifest_paths[@]}"; do
|
||||
"$PYTHON" - "$manifest_path" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
text = path.read_text()
|
||||
text, count = re.subn(
|
||||
r'(?m)^(\s*version=)["\'][^"\']+["\'](,?\s*)$',
|
||||
rf'\1"{new_version}"\2',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if count == 0:
|
||||
text, count = re.subn(
|
||||
r'(?m)^(MODULE_VERSION\s*=\s*)["\'][^"\']+["\'](\s*)$',
|
||||
rf'\1"{new_version}"\2',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if count != 1:
|
||||
raise SystemExit(f"could not update ModuleManifest.version in {path}")
|
||||
path.write_text(text)
|
||||
PYCODE
|
||||
done
|
||||
}
|
||||
|
||||
update_webui_package() {
|
||||
local repo="$1"
|
||||
local project_name="$2"
|
||||
local version="$3"
|
||||
local package_path=""
|
||||
local release_package_path="$repo/webui/package.release.json"
|
||||
|
||||
for package_path in "$repo/package.json" "$repo/webui/package.json"; do
|
||||
[[ -f "$package_path" ]] || continue
|
||||
"$PYTHON" - "$package_path" "$project_name" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
project_name = sys.argv[2]
|
||||
new_version = sys.argv[3]
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
data["version"] = new_version
|
||||
if project_name != "govoplan-core":
|
||||
peers = data.setdefault("peerDependencies", {})
|
||||
if "@govoplan/core-webui" in peers:
|
||||
peers["@govoplan/core-webui"] = f"^{new_version}"
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
done
|
||||
|
||||
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
||||
"$PYTHON" - "$release_package_path" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
data["version"] = new_version
|
||||
dependencies = data.setdefault("dependencies", {})
|
||||
for name, spec in list(dependencies.items()):
|
||||
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
|
||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
fi
|
||||
}
|
||||
|
||||
update_version_files() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
local project_name="${PROJECT_NAMES[$repo]}"
|
||||
|
||||
update_pyproject "$repo" "$version"
|
||||
update_manifest_version "$repo" "$project_name" "$version"
|
||||
update_webui_package "$repo" "$project_name" "$version"
|
||||
}
|
||||
|
||||
refresh_development_webui_lock() {
|
||||
[[ -f "$ROOT/webui/package.json" ]] || return 0
|
||||
[[ -f "$ROOT/webui/package-lock.json" ]] || return 0
|
||||
|
||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$ROOT/webui" install --package-lock-only --ignore-scripts
|
||||
}
|
||||
|
||||
generate_release_lock() {
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 0 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
[[ -x "$ROOT/scripts/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $ROOT/scripts/generate-release-lock.sh"
|
||||
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
||||
}
|
||||
|
||||
run_migration_release_audit() {
|
||||
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
||||
local command=("$PYTHON" "$audit_script")
|
||||
|
||||
case "$MIGRATION_AUDIT_MODE" in
|
||||
skip)
|
||||
echo "Migration release audit: skipped"
|
||||
return
|
||||
;;
|
||||
strict)
|
||||
command+=("--strict")
|
||||
;;
|
||||
auto)
|
||||
command+=("--strict-if-baseline")
|
||||
;;
|
||||
warn)
|
||||
;;
|
||||
*)
|
||||
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
||||
;;
|
||||
esac
|
||||
|
||||
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
||||
run "${command[@]}"
|
||||
}
|
||||
|
||||
print_command() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
run() {
|
||||
print_command "$@"
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_for_bump() {
|
||||
local answer=""
|
||||
|
||||
if [[ -n "$BUMP" || -n "$TARGET_VERSION" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
fail "pass --bump major|minor|subversion or --version x.y.z for non-interactive use"
|
||||
fi
|
||||
|
||||
echo "Select version increase:"
|
||||
echo " 1) major X.Y.Z -> (X+1).0.0"
|
||||
echo " 2) minor X.Y.Z -> X.(Y+1).0"
|
||||
echo " 3) subversion X.Y.Z -> X.Y.(Z+1)"
|
||||
printf 'Choice [subversion]: '
|
||||
read -r answer
|
||||
|
||||
case "${answer,,}" in
|
||||
""|3|subversion|patch|sub|sub-version)
|
||||
BUMP="subversion"
|
||||
;;
|
||||
1|major)
|
||||
BUMP="major"
|
||||
;;
|
||||
2|minor)
|
||||
BUMP="minor"
|
||||
;;
|
||||
*)
|
||||
fail "unknown version bump: $answer"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
confirm_release() {
|
||||
local answer=""
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 || "$YES" -eq 1 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
fail "refusing to commit, tag, and push without confirmation; pass --yes for non-interactive use"
|
||||
fi
|
||||
|
||||
printf 'Commit all changes, tag all repos, and push to %s? [y/N] ' "$REMOTE"
|
||||
read -r answer
|
||||
case "${answer,,}" in
|
||||
y|yes)
|
||||
;;
|
||||
*)
|
||||
echo "Aborted."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--bump)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BUMP="$(normalize_bump "$2")" || fail "unknown bump kind: $2"
|
||||
shift 2
|
||||
;;
|
||||
--version)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
TARGET_VERSION="$2"
|
||||
shift 2
|
||||
;;
|
||||
-r|--remote)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-b|--branch)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-m|--message)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
COMMIT_MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--tag-message)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
TAG_MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--skip-release-lock)
|
||||
GENERATE_RELEASE_LOCK=0
|
||||
shift
|
||||
;;
|
||||
--warn-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="warn"
|
||||
shift
|
||||
;;
|
||||
--strict-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="strict"
|
||||
shift
|
||||
;;
|
||||
--skip-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="skip"
|
||||
shift
|
||||
;;
|
||||
--publish-web-catalog)
|
||||
PUBLISH_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--catalog-channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-public-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_PUBLIC_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web-catalog)
|
||||
BUILD_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-y|--yes)
|
||||
YES=1
|
||||
shift
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
||||
fail "use either --bump or --version, not both"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
|
||||
fail "--publish-web-catalog requires at least one --catalog-signing-key"
|
||||
fi
|
||||
|
||||
prompt_for_bump
|
||||
|
||||
if ! command -v "$PYTHON" >/dev/null 2>&1; then
|
||||
fail "Python interpreter not found: $PYTHON"
|
||||
fi
|
||||
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
|
||||
fail "npm executable not found: $NPM_BIN"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT"
|
||||
[[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh"
|
||||
fi
|
||||
|
||||
declare -A PROJECT_NAMES
|
||||
declare -A OLD_VERSIONS
|
||||
declare -A BRANCHES
|
||||
declare -A REPO_KINDS
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
||||
|
||||
if [[ -f "$repo/pyproject.toml" ]]; then
|
||||
REPO_KINDS["$repo"]="package"
|
||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
||||
|
||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
||||
else
|
||||
REPO_KINDS["$repo"]="tag-only"
|
||||
PROJECT_NAMES["$repo"]="$(basename "$repo")"
|
||||
OLD_VERSIONS["$repo"]="tag-only"
|
||||
fi
|
||||
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
BRANCHES["$repo"]="$BRANCH"
|
||||
else
|
||||
BRANCHES["$repo"]="$(git -C "$repo" symbolic-ref --quiet --short HEAD || true)"
|
||||
fi
|
||||
|
||||
[[ -n "${BRANCHES[$repo]}" ]] || fail "cannot infer branch for $repo; pass --branch <name>"
|
||||
git -C "$repo" check-ref-format "refs/heads/${BRANCHES[$repo]}" || fail "invalid branch name ${BRANCHES[$repo]} for $repo"
|
||||
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
||||
|
||||
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
||||
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
|
||||
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
||||
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$TARGET_VERSION" ]]; then
|
||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
||||
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
||||
fi
|
||||
done
|
||||
TARGET_VERSION="$(bump_version "$BASE_VERSION" "$BUMP")"
|
||||
fi
|
||||
|
||||
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
||||
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
|
||||
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
|
||||
fi
|
||||
done
|
||||
|
||||
TAG="v$TARGET_VERSION"
|
||||
COMMIT_MESSAGE="${COMMIT_MESSAGE:-Release $TAG}"
|
||||
TAG_MESSAGE="${TAG_MESSAGE:-$COMMIT_MESSAGE}"
|
||||
|
||||
git -C "$ROOT" check-ref-format "refs/tags/$TAG" || fail "invalid tag name: $TAG"
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
if git -C "$repo" rev-parse --quiet --verify "refs/tags/$TAG" >/dev/null; then
|
||||
fail "local tag already exists in ${PROJECT_NAMES[$repo]}: $TAG"
|
||||
fi
|
||||
|
||||
set +e
|
||||
timeout "$GIT_REMOTE_TIMEOUT" git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
|
||||
ls_remote_status=$?
|
||||
set -e
|
||||
|
||||
if [[ "$ls_remote_status" -eq 0 ]]; then
|
||||
fail "remote tag already exists for ${PROJECT_NAMES[$repo]} on $REMOTE: $TAG"
|
||||
elif [[ "$ls_remote_status" -ne 2 ]]; then
|
||||
fail "could not check remote tags for ${PROJECT_NAMES[$repo]} on $REMOTE"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Release plan:"
|
||||
echo " version: $TARGET_VERSION"
|
||||
echo " tag: $TAG"
|
||||
echo " remote: $REMOTE"
|
||||
echo " commit message: $COMMIT_MESSAGE"
|
||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
||||
else
|
||||
echo " release lock: skipped"
|
||||
fi
|
||||
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||
else
|
||||
echo " web catalog: skipped"
|
||||
fi
|
||||
echo
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
||||
else
|
||||
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
|
||||
fi
|
||||
git -C "$repo" status --short
|
||||
echo
|
||||
done
|
||||
|
||||
run_migration_release_audit
|
||||
|
||||
confirm_release
|
||||
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Would update version files in $repo to $TARGET_VERSION"
|
||||
else
|
||||
update_version_files "$repo" "$TARGET_VERSION"
|
||||
fi
|
||||
done
|
||||
|
||||
refresh_development_webui_lock
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$repo" diff --cached --quiet; then
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||
echo "No staged changes for ${PROJECT_NAMES[$repo]}; version is already $TARGET_VERSION, tagging current HEAD."
|
||||
else
|
||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
||||
fi
|
||||
else
|
||||
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
|
||||
fi
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
|
||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
done
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||
done
|
||||
|
||||
generate_release_lock
|
||||
|
||||
run git -C "$ROOT" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$ROOT" diff --cached --quiet; then
|
||||
fail "no staged changes for ${PROJECT_NAMES[$ROOT]} after version bump"
|
||||
fi
|
||||
fi
|
||||
|
||||
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS=(
|
||||
"$ROOT/scripts/publish-release-catalog.sh"
|
||||
--version "$TARGET_VERSION"
|
||||
--channel "$CATALOG_CHANNEL"
|
||||
--expires-days "$CATALOG_EXPIRES_DAYS"
|
||||
--core-root "$ROOT"
|
||||
--web-root "$WEB_ROOT"
|
||||
--public-base-url "$CATALOG_PUBLIC_URL"
|
||||
--npm "$NPM_BIN"
|
||||
--commit
|
||||
--tag
|
||||
--push
|
||||
--remote "$REMOTE"
|
||||
)
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
CATALOG_ARGS+=(--branch "$BRANCH")
|
||||
fi
|
||||
if [[ -n "$CATALOG_SEQUENCE" ]]; then
|
||||
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
|
||||
fi
|
||||
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--build-web)
|
||||
fi
|
||||
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
|
||||
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--dry-run)
|
||||
fi
|
||||
run "${CATALOG_ARGS[@]}"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
||||
else
|
||||
echo "Released $TAG across all GovOPlaN repos."
|
||||
fi
|
||||
@@ -1,422 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Migration:
|
||||
owner: str
|
||||
path: Path
|
||||
revision: str
|
||||
down_revisions: tuple[str, ...]
|
||||
branch_labels: tuple[str, ...]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workspace-root",
|
||||
type=Path,
|
||||
default=ROOT.parent,
|
||||
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-file",
|
||||
type=Path,
|
||||
default=DEFAULT_BASELINE_FILE,
|
||||
help="JSON file recording released migration heads.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Fail when current heads are not recorded in the latest release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-if-baseline",
|
||||
action="store_true",
|
||||
help="Run in strict mode only after at least one release baseline is recorded.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--record-release",
|
||||
metavar="VERSION",
|
||||
help="Record the current reviewed migration heads as a release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--replace-release",
|
||||
action="store_true",
|
||||
help="Replace an existing release entry when used with --record-release.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--squash-plan",
|
||||
action="store_true",
|
||||
help="Print the reviewed/manual squash checklist for the current graph.",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
args = parser.parse_args()
|
||||
|
||||
migrations = discover_migrations(args.workspace_root)
|
||||
baseline = load_baseline_file(args.baseline_file)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.record_release:
|
||||
if report["graph_errors"]:
|
||||
print_text_report(report)
|
||||
return 1
|
||||
baseline = record_release_baseline(
|
||||
baseline,
|
||||
report,
|
||||
release=args.record_release,
|
||||
replace=args.replace_release,
|
||||
)
|
||||
write_baseline_file(args.baseline_file, baseline)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.squash_plan:
|
||||
print_squash_plan(report)
|
||||
elif args.json:
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
else:
|
||||
print_text_report(report)
|
||||
|
||||
has_graph_errors = bool(report["graph_errors"])
|
||||
has_strict_errors = bool(report["strict_errors"])
|
||||
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
||||
if has_graph_errors or (strict_required and has_strict_errors):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
||||
migrations: list[Migration] = []
|
||||
for versions_dir in _migration_version_dirs(workspace_root):
|
||||
owner = owner_for_versions_dir(versions_dir)
|
||||
for path in sorted(versions_dir.glob("*.py")):
|
||||
if path.name == "__init__.py":
|
||||
continue
|
||||
migration = parse_migration_file(owner, path)
|
||||
if migration is not None:
|
||||
migrations.append(migration)
|
||||
return migrations
|
||||
|
||||
|
||||
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
||||
candidates = [ROOT / "alembic" / "versions"]
|
||||
for repo in sorted(workspace_root.glob("govoplan-*")):
|
||||
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
||||
seen: set[Path] = set()
|
||||
result: list[Path] = []
|
||||
for candidate in candidates:
|
||||
resolved = candidate.resolve()
|
||||
if resolved in seen or not candidate.is_dir():
|
||||
continue
|
||||
seen.add(resolved)
|
||||
result.append(candidate)
|
||||
return result
|
||||
|
||||
|
||||
def owner_for_versions_dir(versions_dir: Path) -> str:
|
||||
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
||||
return "govoplan-core"
|
||||
for parent in versions_dir.parents:
|
||||
if parent.name.startswith("govoplan-"):
|
||||
return parent.name
|
||||
return versions_dir.parent.name
|
||||
|
||||
|
||||
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
values: dict[str, Any] = {}
|
||||
for statement in tree.body:
|
||||
if isinstance(statement, ast.Assign):
|
||||
for target in statement.targets:
|
||||
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
||||
values[target.id] = ast.literal_eval(statement.value)
|
||||
elif (
|
||||
isinstance(statement, ast.AnnAssign)
|
||||
and isinstance(statement.target, ast.Name)
|
||||
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
||||
and statement.value is not None
|
||||
):
|
||||
values[statement.target.id] = ast.literal_eval(statement.value)
|
||||
revision = values.get("revision")
|
||||
if not isinstance(revision, str):
|
||||
return None
|
||||
return Migration(
|
||||
owner=owner,
|
||||
path=path,
|
||||
revision=revision,
|
||||
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
||||
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value if item is not None)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def load_baseline_file(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"version": 1, "releases": [], "_missing": True}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit(f"{path} must contain a JSON object")
|
||||
releases = payload.get("releases")
|
||||
if not isinstance(releases, list):
|
||||
raise SystemExit(f"{path} must contain a releases list")
|
||||
return payload
|
||||
|
||||
|
||||
def build_report(
|
||||
migrations: list[Migration],
|
||||
*,
|
||||
baseline: dict[str, Any],
|
||||
baseline_file: Path,
|
||||
workspace_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
revisions: dict[str, Migration] = {}
|
||||
graph_errors: list[str] = []
|
||||
for migration in migrations:
|
||||
existing = revisions.get(migration.revision)
|
||||
if existing is not None:
|
||||
graph_errors.append(
|
||||
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
||||
)
|
||||
revisions[migration.revision] = migration
|
||||
|
||||
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
||||
for migration in migrations:
|
||||
for down_revision in migration.down_revisions:
|
||||
if down_revision not in revisions:
|
||||
graph_errors.append(
|
||||
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
||||
)
|
||||
|
||||
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
||||
current_head_entries = [
|
||||
{
|
||||
"owner": revisions[revision].owner,
|
||||
"revision": revision,
|
||||
}
|
||||
for revision in current_heads
|
||||
]
|
||||
owner_rows = []
|
||||
for owner in sorted({migration.owner for migration in migrations}):
|
||||
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
||||
owner_revisions = {migration.revision for migration in owner_migrations}
|
||||
owner_referenced = {
|
||||
revision
|
||||
for migration in owner_migrations
|
||||
for revision in migration.down_revisions
|
||||
if revision in owner_revisions
|
||||
}
|
||||
owner_heads = sorted(owner_revisions - owner_referenced)
|
||||
owner_rows.append(
|
||||
{
|
||||
"owner": owner,
|
||||
"migrations": len(owner_migrations),
|
||||
"heads": owner_heads,
|
||||
}
|
||||
)
|
||||
|
||||
releases = baseline.get("releases") or []
|
||||
latest_release = releases[-1] if releases else None
|
||||
latest_heads = _latest_release_heads(latest_release)
|
||||
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
||||
|
||||
strict_errors: list[str] = []
|
||||
if baseline.get("_missing"):
|
||||
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
||||
if not releases:
|
||||
strict_errors.append("no released migration baseline has been recorded yet")
|
||||
if unrecorded_heads:
|
||||
strict_errors.append(
|
||||
"current migration heads are not recorded in the latest release baseline: "
|
||||
+ ", ".join(unrecorded_heads)
|
||||
)
|
||||
return {
|
||||
"workspace_root": str(workspace_root),
|
||||
"baseline_file": str(baseline_file),
|
||||
"baseline_file_missing": bool(baseline.get("_missing")),
|
||||
"release_count": len(releases),
|
||||
"latest_release": latest_release,
|
||||
"owners": owner_rows,
|
||||
"current_heads": current_heads,
|
||||
"current_head_entries": current_head_entries,
|
||||
"graph_errors": graph_errors,
|
||||
"strict_errors": strict_errors,
|
||||
}
|
||||
|
||||
|
||||
def _latest_release_heads(latest_release: Any) -> set[str]:
|
||||
if not isinstance(latest_release, dict):
|
||||
return set()
|
||||
heads = latest_release.get("heads") or []
|
||||
graph_heads = latest_release.get("graph_heads") or []
|
||||
result: set[str] = set()
|
||||
for head_list in (heads, graph_heads):
|
||||
if not isinstance(head_list, list):
|
||||
continue
|
||||
for entry in head_list:
|
||||
if isinstance(entry, str):
|
||||
result.add(entry)
|
||||
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
||||
result.add(entry["revision"])
|
||||
owner_heads = latest_release.get("owner_heads") or []
|
||||
if isinstance(owner_heads, list):
|
||||
for entry in owner_heads:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
revisions = entry.get("revisions") or []
|
||||
if isinstance(revisions, str):
|
||||
result.add(revisions)
|
||||
elif isinstance(revisions, list):
|
||||
result.update(revision for revision in revisions if isinstance(revision, str))
|
||||
return result
|
||||
|
||||
|
||||
def record_release_baseline(
|
||||
baseline: dict[str, Any],
|
||||
report: dict[str, Any],
|
||||
*,
|
||||
release: str,
|
||||
replace: bool,
|
||||
) -> dict[str, Any]:
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in baseline.items()
|
||||
if not key.startswith("_")
|
||||
}
|
||||
payload.setdefault("version", 1)
|
||||
releases = list(payload.get("releases") or [])
|
||||
existing_index = next(
|
||||
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
||||
None,
|
||||
)
|
||||
if existing_index is not None and not replace:
|
||||
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
||||
|
||||
entry = {
|
||||
"release": release,
|
||||
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"squash_policy": "reviewed-manual",
|
||||
"heads": report["current_head_entries"],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": owner["owner"],
|
||||
"revisions": owner["heads"],
|
||||
}
|
||||
for owner in report["owners"]
|
||||
],
|
||||
}
|
||||
if existing_index is None:
|
||||
releases.append(entry)
|
||||
else:
|
||||
releases[existing_index] = entry
|
||||
payload["releases"] = releases
|
||||
return payload
|
||||
|
||||
|
||||
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def print_text_report(report: dict[str, Any]) -> None:
|
||||
print("Migration release audit")
|
||||
print(f" workspace: {report['workspace_root']}")
|
||||
print(f" baseline file: {report['baseline_file']}")
|
||||
print(f" recorded releases: {report['release_count']}")
|
||||
print()
|
||||
print("Owners:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
||||
print()
|
||||
print("Current heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
|
||||
if report["graph_errors"]:
|
||||
print()
|
||||
print("Graph errors:")
|
||||
for error in report["graph_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
if report["strict_errors"]:
|
||||
print()
|
||||
print("Release baseline warnings:")
|
||||
for error in report["strict_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
|
||||
def print_squash_plan(report: dict[str, Any]) -> None:
|
||||
print("Manual migration squash plan")
|
||||
print()
|
||||
print("Policy:")
|
||||
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
||||
print(" - Squash only unreleased development migrations.")
|
||||
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
||||
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
||||
print()
|
||||
print("Current owner heads:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" - {owner['owner']}: {heads}")
|
||||
print()
|
||||
print("Current Alembic graph heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" - {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
print()
|
||||
print("Release steps:")
|
||||
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
||||
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
||||
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
||||
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
||||
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -94,6 +94,28 @@ class UserInfo(BaseModel):
|
||||
ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences)
|
||||
|
||||
|
||||
class AuthSessionUserInfo(BaseModel):
|
||||
id: str
|
||||
account_id: str
|
||||
email: str
|
||||
display_name: str | None = None
|
||||
tenant_display_name: str | None = None
|
||||
is_tenant_admin: bool = False
|
||||
password_reset_required: bool = False
|
||||
|
||||
|
||||
class AuthSessionResponse(BaseModel):
|
||||
authenticated: bool = True
|
||||
auth_method: Literal["session", "api_key"] = "session"
|
||||
user: AuthSessionUserInfo
|
||||
# Backwards-compatible alias for the active tenant.
|
||||
tenant: TenantInfo
|
||||
active_tenant: TenantInfo
|
||||
session_id: str | None = None
|
||||
api_key_id: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
|
||||
|
||||
class ProfileUpdateRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -143,9 +165,43 @@ class PrincipalContextInfo(BaseModel):
|
||||
display_name: str | None = None
|
||||
|
||||
|
||||
class AuthShellResponse(BaseModel):
|
||||
user: UserInfo
|
||||
# Backwards-compatible alias for the active tenant.
|
||||
tenant: TenantInfo
|
||||
active_tenant: TenantInfo
|
||||
tenants: list[TenantMembershipInfo] = Field(default_factory=list)
|
||||
scopes: list[str]
|
||||
principal: PrincipalContextInfo | None = None
|
||||
profile_loaded: bool = False
|
||||
roles_loaded: bool = False
|
||||
groups_loaded: bool = False
|
||||
|
||||
|
||||
class AuthProfileResponse(BaseModel):
|
||||
user: UserInfo
|
||||
# Backwards-compatible alias for the active tenant.
|
||||
tenant: TenantInfo
|
||||
active_tenant: TenantInfo
|
||||
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||
default_language: str = "en"
|
||||
profile_loaded: bool = True
|
||||
|
||||
|
||||
class AuthRolesResponse(BaseModel):
|
||||
roles: list[RoleInfo] = Field(default_factory=list)
|
||||
roles_loaded: bool = True
|
||||
|
||||
|
||||
class AuthGroupsResponse(BaseModel):
|
||||
groups: list[GroupInfo] = Field(default_factory=list)
|
||||
groups_loaded: bool = True
|
||||
|
||||
|
||||
class LoginResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
token_type: str = "bearer" # noqa: S105 - OAuth token type, not a credential.
|
||||
expires_at: datetime
|
||||
user: UserInfo
|
||||
# Backwards-compatible alias for the active tenant.
|
||||
@@ -159,6 +215,9 @@ class LoginResponse(BaseModel):
|
||||
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||
default_language: str = "en"
|
||||
profile_loaded: bool = True
|
||||
roles_loaded: bool = True
|
||||
groups_loaded: bool = True
|
||||
|
||||
|
||||
class MeResponse(BaseModel):
|
||||
@@ -174,3 +233,6 @@ class MeResponse(BaseModel):
|
||||
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||
default_language: str = "en"
|
||||
profile_loaded: bool = True
|
||||
roles_loaded: bool = True
|
||||
groups_loaded: bool = True
|
||||
|
||||
@@ -20,30 +20,13 @@ from govoplan_core.core.events import (
|
||||
)
|
||||
from govoplan_core.core.runtime import get_registry
|
||||
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
||||
from govoplan_core.security.redaction import redact_secret_values
|
||||
|
||||
|
||||
AUDIT_MODULE_ID = "audit"
|
||||
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
|
||||
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
|
||||
|
||||
SENSITIVE_DETAIL_KEYS = {
|
||||
"password",
|
||||
"smtp_password",
|
||||
"imap_password",
|
||||
"secret",
|
||||
"api_key",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"authorization",
|
||||
"cookie",
|
||||
"credentials",
|
||||
"credential",
|
||||
"private_key",
|
||||
"claim_token",
|
||||
"build_token",
|
||||
}
|
||||
|
||||
_ACTION_MODULE_PREFIXES = {
|
||||
"api_key": "access",
|
||||
"configuration_change": "access",
|
||||
@@ -90,17 +73,7 @@ def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]:
|
||||
|
||||
|
||||
def _sanitize_details(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).lower() in SENSITIVE_DETAIL_KEYS:
|
||||
sanitized[key] = "<redacted>"
|
||||
else:
|
||||
sanitized[key] = _sanitize_details(item)
|
||||
return sanitized
|
||||
if isinstance(value, list):
|
||||
return [_sanitize_details(item) for item in value]
|
||||
return value
|
||||
return redact_secret_values(value)
|
||||
|
||||
|
||||
def audit_operation_context(
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""Core auth dependency facade.
|
||||
|
||||
Routers depend on this module instead of a concrete access-provider package.
|
||||
@@ -7,6 +5,8 @@ The active auth module provides the request principal through the platform
|
||||
capability registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import Depends, Header, HTTPException, Request, status
|
||||
|
||||
@@ -3,8 +3,11 @@ from __future__ import annotations
|
||||
from celery import Celery
|
||||
|
||||
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider
|
||||
from govoplan_core.core.calendar import CAPABILITY_CALENDAR_OUTBOX, CalendarOutboxProvider
|
||||
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||
from govoplan_core.core.modules import ModuleContext
|
||||
from govoplan_core.core.notifications import CAPABILITY_NOTIFICATIONS_DISPATCH, NotificationDispatchProvider
|
||||
from govoplan_core.core.registry import PlatformRegistry
|
||||
from govoplan_core.core.runtime import configure_runtime
|
||||
from govoplan_core.settings import settings
|
||||
from govoplan_core.db.session import configure_database
|
||||
@@ -13,7 +16,7 @@ from govoplan_core.server.registry import available_module_manifests, build_plat
|
||||
configure_database(settings.database_url)
|
||||
|
||||
celery = Celery(
|
||||
"multimailer",
|
||||
"govoplan",
|
||||
broker=settings.redis_url,
|
||||
backend=settings.redis_url,
|
||||
)
|
||||
@@ -21,21 +24,31 @@ celery = Celery(
|
||||
celery.conf.update(
|
||||
task_default_queue="default",
|
||||
task_routes={
|
||||
"multimailer.send_email": {"queue": "send_email"},
|
||||
"multimailer.append_sent": {"queue": "append_sent"},
|
||||
"govoplan.campaigns.send_email": {"queue": "send_email"},
|
||||
"govoplan.campaigns.append_sent": {"queue": "append_sent"},
|
||||
"govoplan.notifications.deliver": {"queue": "notifications"},
|
||||
"govoplan.notifications.deliver_pending": {"queue": "notifications"},
|
||||
"govoplan.calendar.dispatch_outbox": {"queue": "calendar"},
|
||||
},
|
||||
worker_prefetch_multiplier=1,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
beat_schedule={
|
||||
"calendar-outbox-every-minute": {
|
||||
"task": "govoplan.calendar.dispatch_outbox",
|
||||
"schedule": 60.0,
|
||||
"args": (None, 100),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@celery.task(name="multimailer.ping")
|
||||
@celery.task(name="govoplan.ping")
|
||||
def ping():
|
||||
return "pong"
|
||||
|
||||
|
||||
def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
||||
def _platform_registry() -> PlatformRegistry:
|
||||
raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules)
|
||||
candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules)
|
||||
available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True)
|
||||
@@ -44,13 +57,36 @@ def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
||||
context = ModuleContext(registry=registry, settings=settings)
|
||||
configure_runtime(context)
|
||||
registry.configure_capability_context(context)
|
||||
return registry
|
||||
|
||||
|
||||
def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
||||
registry = _platform_registry()
|
||||
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_DELIVERY_TASKS)
|
||||
if not isinstance(capability, CampaignDeliveryTaskProvider):
|
||||
raise RuntimeError("Campaign delivery task capability is invalid")
|
||||
return capability
|
||||
|
||||
|
||||
@celery.task(name="multimailer.send_email", bind=True, max_retries=0)
|
||||
def _notification_dispatch() -> NotificationDispatchProvider:
|
||||
registry = _platform_registry()
|
||||
capability = registry.require_capability(CAPABILITY_NOTIFICATIONS_DISPATCH)
|
||||
if not isinstance(capability, NotificationDispatchProvider):
|
||||
raise RuntimeError("Notification dispatch capability is invalid")
|
||||
return capability
|
||||
|
||||
|
||||
def _calendar_outbox() -> CalendarOutboxProvider | None:
|
||||
registry = _platform_registry()
|
||||
if not registry.has_capability(CAPABILITY_CALENDAR_OUTBOX):
|
||||
return None
|
||||
capability = registry.require_capability(CAPABILITY_CALENDAR_OUTBOX)
|
||||
if not isinstance(capability, CalendarOutboxProvider):
|
||||
raise RuntimeError("Calendar outbox capability is invalid")
|
||||
return capability
|
||||
|
||||
|
||||
@celery.task(name="govoplan.campaigns.send_email", bind=True, max_retries=0)
|
||||
def send_email(self, job_id: str):
|
||||
"""Send one explicitly queued campaign job.
|
||||
|
||||
@@ -65,7 +101,7 @@ def send_email(self, job_id: str):
|
||||
return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True))
|
||||
|
||||
|
||||
@celery.task(name="multimailer.append_sent", bind=True, max_retries=None)
|
||||
@celery.task(name="govoplan.campaigns.append_sent", bind=True, max_retries=None)
|
||||
def append_sent(self, job_id: str):
|
||||
"""Append the exact sent MIME to the configured IMAP Sent folder."""
|
||||
|
||||
@@ -78,3 +114,38 @@ def append_sent(self, job_id: str):
|
||||
if getattr(exc, "temporary", None) is True:
|
||||
raise self.retry(exc=exc, countdown=300)
|
||||
raise
|
||||
|
||||
|
||||
@celery.task(name="govoplan.notifications.deliver", bind=True, max_retries=0)
|
||||
def deliver_notification(self, notification_id: str):
|
||||
from govoplan_core.db.session import get_database
|
||||
|
||||
with get_database().SessionLocal() as session:
|
||||
result = dict(_notification_dispatch().deliver_notification(session, notification_id=notification_id))
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
|
||||
@celery.task(name="govoplan.notifications.deliver_pending", bind=True, max_retries=0)
|
||||
def deliver_pending_notifications(self, tenant_id: str | None = None, limit: int = 50):
|
||||
from govoplan_core.db.session import get_database
|
||||
|
||||
with get_database().SessionLocal() as session:
|
||||
result = dict(_notification_dispatch().deliver_pending(session, tenant_id=tenant_id, limit=limit))
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
|
||||
@celery.task(name="govoplan.calendar.dispatch_outbox", bind=True, max_retries=0)
|
||||
def dispatch_calendar_outbox(self, tenant_id: str | None = None, limit: int = 50):
|
||||
"""Drain durable Calendar operations; retry timing lives in the database."""
|
||||
|
||||
from govoplan_core.db.session import get_database
|
||||
|
||||
with get_database().SessionLocal() as session:
|
||||
provider = _calendar_outbox()
|
||||
if provider is None:
|
||||
return {"processed": 0, "succeeded": 0, "retrying": 0, "failed": 0, "operations": []}
|
||||
result = dict(provider.dispatch_due(session, tenant_id=tenant_id, limit=limit))
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
@@ -19,6 +19,12 @@ from govoplan_core.settings import settings
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
|
||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
|
||||
parser.add_argument(
|
||||
"--migration-track",
|
||||
default=settings.migration_track,
|
||||
choices=("release", "dev"),
|
||||
help="Alembic migration track to load: release uses squashed release shortcuts; dev uses detailed development chains.",
|
||||
)
|
||||
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
|
||||
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
|
||||
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
|
||||
@@ -42,6 +48,7 @@ def main() -> None:
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_module_order=migration_order,
|
||||
migration_track=args.migration_track,
|
||||
)
|
||||
_run_migration_tasks(
|
||||
task_records,
|
||||
|
||||
@@ -5,7 +5,6 @@ import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from govoplan_core.core.module_installer import (
|
||||
ModuleInstallerError,
|
||||
@@ -27,6 +26,13 @@ from govoplan_core.core.module_installer import (
|
||||
update_module_installer_request,
|
||||
update_module_installer_daemon_status,
|
||||
)
|
||||
from govoplan_core.core.module_installer_notifications import (
|
||||
build_runtime_notification_registry,
|
||||
emit_module_installer_notification,
|
||||
installer_notification_body,
|
||||
installer_notification_priority,
|
||||
installer_notification_subject,
|
||||
)
|
||||
from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics
|
||||
from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog
|
||||
from govoplan_core.core.module_management import (
|
||||
@@ -39,7 +45,7 @@ from govoplan_core.server.registry import available_module_manifests
|
||||
from govoplan_core.settings import settings
|
||||
|
||||
|
||||
def main() -> int:
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Preflight, apply, or roll back a GovOPlaN module package install plan.")
|
||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.")
|
||||
parser.add_argument("--runtime-dir", type=Path, help="Directory for installer locks and run snapshots.")
|
||||
@@ -95,202 +101,295 @@ def main() -> int:
|
||||
parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.")
|
||||
parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.")
|
||||
parser.add_argument("--license-notes", help="Optional operator note for --issue-license.")
|
||||
args = parser.parse_args()
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _build_parser().parse_args()
|
||||
runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url)
|
||||
try:
|
||||
if args.issue_license:
|
||||
if not args.license_id or not args.license_subject or not args.license_valid_until:
|
||||
raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.")
|
||||
if not args.license_signing_key_id or not args.license_signing_private_key:
|
||||
raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.")
|
||||
if not [item for item in args.license_feature if item.strip()]:
|
||||
raise ModuleInstallerError("--issue-license requires at least one --license-feature.")
|
||||
try:
|
||||
path = issue_module_license(
|
||||
path=args.issue_license,
|
||||
license_id=args.license_id,
|
||||
subject=args.license_subject,
|
||||
features=args.license_feature,
|
||||
valid_from=args.license_valid_from,
|
||||
valid_until=args.license_valid_until,
|
||||
signing_key_id=args.license_signing_key_id,
|
||||
signing_private_key_path=args.license_signing_private_key,
|
||||
issuer=args.license_issuer,
|
||||
notes=args.license_notes,
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise ModuleInstallerError(str(exc)) from exc
|
||||
_print_result(
|
||||
{
|
||||
"issued": True,
|
||||
"path": str(path),
|
||||
"license_id": args.license_id,
|
||||
"subject": args.license_subject,
|
||||
"features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())),
|
||||
"valid_until": args.license_valid_until,
|
||||
"key_id": args.license_signing_key_id,
|
||||
},
|
||||
output_format=args.format,
|
||||
)
|
||||
return 0
|
||||
if args.validate_license is not None:
|
||||
path = Path(args.validate_license).expanduser() if args.validate_license else None
|
||||
result = module_license_diagnostics(
|
||||
path,
|
||||
require_trusted=args.require_trusted_license,
|
||||
trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"),
|
||||
required_features=args.license_required_feature,
|
||||
)
|
||||
_print_result(result, output_format=args.format)
|
||||
return 0 if result.get("valid") and not result.get("missing_features") else 1
|
||||
if args.sign_package_catalog:
|
||||
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
||||
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
||||
path = sign_module_package_catalog(
|
||||
path=args.sign_package_catalog,
|
||||
key_id=args.catalog_signing_key_id,
|
||||
private_key_path=args.catalog_signing_private_key,
|
||||
output_path=args.catalog_output,
|
||||
)
|
||||
_print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format)
|
||||
return 0
|
||||
if args.validate_package_catalog is not None:
|
||||
path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None
|
||||
result = validate_module_package_catalog(
|
||||
path,
|
||||
require_trusted=args.require_signed_catalog,
|
||||
approved_channels=tuple(args.approved_catalog_channel),
|
||||
trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"),
|
||||
)
|
||||
_print_result(result, output_format=args.format)
|
||||
return 0 if result.get("valid") else 1
|
||||
if args.daemon_status:
|
||||
_print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||
return 0
|
||||
if args.daemon or args.daemon_once:
|
||||
return _run_daemon(args=args, runtime_dir=runtime_dir)
|
||||
if args.list_requests:
|
||||
_print_result({
|
||||
"requests": list(list_module_installer_requests(runtime_dir=runtime_dir)),
|
||||
"daemon": module_installer_daemon_status(runtime_dir=runtime_dir),
|
||||
}, output_format=args.format)
|
||||
return 0
|
||||
if args.show_request:
|
||||
_print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format)
|
||||
return 0
|
||||
if args.cancel_request:
|
||||
_print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format)
|
||||
return 0
|
||||
if args.retry_request:
|
||||
_print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format)
|
||||
return 0
|
||||
if args.enqueue_supervised:
|
||||
request = queue_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
requested_by="cli",
|
||||
options=_request_options_from_args(args),
|
||||
)
|
||||
_print_result(request, output_format=args.format)
|
||||
return 0
|
||||
if args.list_runs:
|
||||
_print_result({
|
||||
"runs": list(list_module_installer_runs(runtime_dir=runtime_dir)),
|
||||
"lock": module_installer_lock_status(runtime_dir=runtime_dir),
|
||||
}, output_format=args.format)
|
||||
return 0
|
||||
if args.show_run:
|
||||
_print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format)
|
||||
return 0
|
||||
if args.lock_status:
|
||||
_print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||
return 0
|
||||
if args.rollback:
|
||||
result = rollback_module_install_run(
|
||||
run_id=args.rollback,
|
||||
runtime_dir=runtime_dir,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_url=str(args.database_url),
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
|
||||
configure_database(str(args.database_url))
|
||||
available = available_module_manifests(ignore_load_errors=True)
|
||||
with get_database().session() as session:
|
||||
configured = configured_enabled_modules(settings.enabled_modules)
|
||||
desired = saved_desired_enabled_modules(session, configured)
|
||||
plan = saved_module_install_plan(session)
|
||||
if args.supervise:
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
database_url=str(args.database_url),
|
||||
runtime_dir=runtime_dir,
|
||||
webui_root=args.webui_root,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
migrate_database=args.migrate,
|
||||
database_backup_command=args.database_backup_command,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_restore_check_command=args.database_restore_check_command,
|
||||
activate_installed_modules=not args.no_activate_installed_modules,
|
||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||
restart_command=args.restart_command,
|
||||
health_url=args.health_url,
|
||||
health_timeout_seconds=args.health_timeout_seconds,
|
||||
health_interval_seconds=args.health_interval_seconds,
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
if args.apply or args.dry_run:
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
database_url=str(args.database_url),
|
||||
runtime_dir=runtime_dir,
|
||||
webui_root=args.webui_root,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
migrate_database=args.migrate,
|
||||
database_backup_command=args.database_backup_command,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_restore_check_command=args.database_restore_check_command,
|
||||
activate_installed_modules=not args.no_activate_installed_modules,
|
||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
|
||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||
|
||||
maintenance = saved_maintenance_mode(session)
|
||||
preflight = module_install_preflight(
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
maintenance_mode=maintenance.enabled,
|
||||
session=session,
|
||||
webui_root=args.webui_root,
|
||||
runtime_dir=runtime_dir,
|
||||
)
|
||||
_print_preflight(preflight.as_dict(), output_format=args.format)
|
||||
return 0 if preflight.allowed else 1
|
||||
return _dispatch_command(args=args, runtime_dir=runtime_dir)
|
||||
except ModuleInstallerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _dispatch_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||
for handler in (
|
||||
_handle_license_command,
|
||||
_handle_catalog_command,
|
||||
_handle_runtime_command,
|
||||
_handle_queue_command,
|
||||
_handle_run_history_command,
|
||||
):
|
||||
result = handler(args=args, runtime_dir=runtime_dir)
|
||||
if result is not None:
|
||||
return result
|
||||
return _handle_install_command(args=args, runtime_dir=runtime_dir)
|
||||
|
||||
|
||||
def _handle_license_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||
del runtime_dir
|
||||
if args.issue_license:
|
||||
return _issue_license_command(args)
|
||||
if args.validate_license is not None:
|
||||
return _validate_license_command(args)
|
||||
return None
|
||||
|
||||
|
||||
def _issue_license_command(args: argparse.Namespace) -> int:
|
||||
if not args.license_id or not args.license_subject or not args.license_valid_until:
|
||||
raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.")
|
||||
if not args.license_signing_key_id or not args.license_signing_private_key:
|
||||
raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.")
|
||||
if not [item for item in args.license_feature if item.strip()]:
|
||||
raise ModuleInstallerError("--issue-license requires at least one --license-feature.")
|
||||
try:
|
||||
path = issue_module_license(
|
||||
path=args.issue_license,
|
||||
license_id=args.license_id,
|
||||
subject=args.license_subject,
|
||||
features=args.license_feature,
|
||||
valid_from=args.license_valid_from,
|
||||
valid_until=args.license_valid_until,
|
||||
signing_key_id=args.license_signing_key_id,
|
||||
signing_private_key_path=args.license_signing_private_key,
|
||||
issuer=args.license_issuer,
|
||||
notes=args.license_notes,
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise ModuleInstallerError(str(exc)) from exc
|
||||
_print_result(
|
||||
{
|
||||
"issued": True,
|
||||
"path": str(path),
|
||||
"license_id": args.license_id,
|
||||
"subject": args.license_subject,
|
||||
"features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())),
|
||||
"valid_until": args.license_valid_until,
|
||||
"key_id": args.license_signing_key_id,
|
||||
},
|
||||
output_format=args.format,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _validate_license_command(args: argparse.Namespace) -> int:
|
||||
path = Path(args.validate_license).expanduser() if args.validate_license else None
|
||||
result = module_license_diagnostics(
|
||||
path,
|
||||
require_trusted=args.require_trusted_license,
|
||||
trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"),
|
||||
required_features=args.license_required_feature,
|
||||
)
|
||||
_print_result(result, output_format=args.format)
|
||||
return 0 if result.get("valid") and not result.get("missing_features") else 1
|
||||
|
||||
|
||||
def _handle_catalog_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||
del runtime_dir
|
||||
if args.sign_package_catalog:
|
||||
return _sign_package_catalog_command(args)
|
||||
if args.validate_package_catalog is not None:
|
||||
return _validate_package_catalog_command(args)
|
||||
return None
|
||||
|
||||
|
||||
def _sign_package_catalog_command(args: argparse.Namespace) -> int:
|
||||
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
||||
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
||||
path = sign_module_package_catalog(
|
||||
path=args.sign_package_catalog,
|
||||
key_id=args.catalog_signing_key_id,
|
||||
private_key_path=args.catalog_signing_private_key,
|
||||
output_path=args.catalog_output,
|
||||
)
|
||||
_print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format)
|
||||
return 0
|
||||
|
||||
|
||||
def _validate_package_catalog_command(args: argparse.Namespace) -> int:
|
||||
path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None
|
||||
result = validate_module_package_catalog(
|
||||
path,
|
||||
require_trusted=args.require_signed_catalog,
|
||||
approved_channels=tuple(args.approved_catalog_channel),
|
||||
trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"),
|
||||
)
|
||||
_print_result(result, output_format=args.format)
|
||||
return 0 if result.get("valid") else 1
|
||||
|
||||
|
||||
def _handle_runtime_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||
if args.daemon_status:
|
||||
_print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||
return 0
|
||||
if args.daemon or args.daemon_once:
|
||||
return _run_daemon(args=args, runtime_dir=runtime_dir)
|
||||
return None
|
||||
|
||||
|
||||
def _handle_queue_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||
if args.list_requests:
|
||||
_print_result({
|
||||
"requests": list(list_module_installer_requests(runtime_dir=runtime_dir)),
|
||||
"daemon": module_installer_daemon_status(runtime_dir=runtime_dir),
|
||||
}, output_format=args.format)
|
||||
return 0
|
||||
if args.show_request:
|
||||
_print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format)
|
||||
return 0
|
||||
if args.cancel_request:
|
||||
_print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format)
|
||||
return 0
|
||||
if args.retry_request:
|
||||
_print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format)
|
||||
return 0
|
||||
if args.enqueue_supervised:
|
||||
request = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="cli", options=_request_options_from_args(args))
|
||||
_print_result(request, output_format=args.format)
|
||||
return 0
|
||||
return None
|
||||
|
||||
|
||||
def _handle_run_history_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||
if args.list_runs:
|
||||
_print_result({
|
||||
"runs": list(list_module_installer_runs(runtime_dir=runtime_dir)),
|
||||
"lock": module_installer_lock_status(runtime_dir=runtime_dir),
|
||||
}, output_format=args.format)
|
||||
return 0
|
||||
if args.show_run:
|
||||
_print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format)
|
||||
return 0
|
||||
if args.lock_status:
|
||||
_print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||
return 0
|
||||
if args.rollback:
|
||||
return _rollback_command(args=args, runtime_dir=runtime_dir)
|
||||
return None
|
||||
|
||||
|
||||
def _rollback_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||
result = rollback_module_install_run(
|
||||
run_id=args.rollback,
|
||||
runtime_dir=runtime_dir,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_url=str(args.database_url),
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
|
||||
|
||||
def _handle_install_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||
configure_database(str(args.database_url))
|
||||
available = available_module_manifests(ignore_load_errors=True)
|
||||
with get_database().session() as session:
|
||||
configured = configured_enabled_modules(settings.enabled_modules)
|
||||
desired = saved_desired_enabled_modules(session, configured)
|
||||
plan = saved_module_install_plan(session)
|
||||
if args.supervise:
|
||||
return _supervise_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||
if args.apply or args.dry_run:
|
||||
return _apply_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||
return _preflight_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||
|
||||
|
||||
def _supervise_install_command(
|
||||
*,
|
||||
args: argparse.Namespace,
|
||||
runtime_dir: Path,
|
||||
session: object,
|
||||
available: object,
|
||||
desired: object,
|
||||
plan: object,
|
||||
) -> int:
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
database_url=str(args.database_url),
|
||||
runtime_dir=runtime_dir,
|
||||
webui_root=args.webui_root,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
migrate_database=args.migrate,
|
||||
database_backup_command=args.database_backup_command,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_restore_check_command=args.database_restore_check_command,
|
||||
activate_installed_modules=not args.no_activate_installed_modules,
|
||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||
restart_command=args.restart_command,
|
||||
health_url=args.health_url,
|
||||
health_timeout_seconds=args.health_timeout_seconds,
|
||||
health_interval_seconds=args.health_interval_seconds,
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
|
||||
|
||||
def _apply_install_command(
|
||||
*,
|
||||
args: argparse.Namespace,
|
||||
runtime_dir: Path,
|
||||
session: object,
|
||||
available: object,
|
||||
desired: object,
|
||||
plan: object,
|
||||
) -> int:
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
database_url=str(args.database_url),
|
||||
runtime_dir=runtime_dir,
|
||||
webui_root=args.webui_root,
|
||||
npm_bin=args.npm_bin,
|
||||
build_webui=args.build_webui,
|
||||
migrate_database=args.migrate,
|
||||
database_backup_command=args.database_backup_command,
|
||||
database_restore_command=args.database_restore_command,
|
||||
database_restore_check_command=args.database_restore_check_command,
|
||||
activate_installed_modules=not args.no_activate_installed_modules,
|
||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
_print_result(result.as_dict(), output_format=args.format)
|
||||
return 0 if result.return_code == 0 else 1
|
||||
|
||||
|
||||
def _preflight_install_command(
|
||||
*,
|
||||
args: argparse.Namespace,
|
||||
runtime_dir: Path,
|
||||
session: object,
|
||||
available: object,
|
||||
desired: object,
|
||||
plan: object,
|
||||
) -> int:
|
||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||
|
||||
maintenance = saved_maintenance_mode(session)
|
||||
preflight = module_install_preflight(
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=desired,
|
||||
desired_enabled=desired,
|
||||
maintenance_mode=maintenance.enabled,
|
||||
session=session,
|
||||
webui_root=args.webui_root,
|
||||
runtime_dir=runtime_dir,
|
||||
)
|
||||
_print_preflight(preflight.as_dict(), output_format=args.format)
|
||||
return 0 if preflight.allowed else 1
|
||||
|
||||
|
||||
def _default_webui_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3] / "webui"
|
||||
|
||||
@@ -355,6 +454,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
||||
request_id = str(request["request_id"])
|
||||
options = request.get("options") if isinstance(request.get("options"), dict) else {}
|
||||
configure_database(str(args.database_url))
|
||||
_emit_installer_daemon_notification(request, event_kind="module_installer.request.running")
|
||||
try:
|
||||
available = available_module_manifests(ignore_load_errors=True)
|
||||
with get_database().session() as session:
|
||||
@@ -384,7 +484,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
||||
health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds),
|
||||
request_context=_request_context(request),
|
||||
)
|
||||
update_module_installer_request(
|
||||
updated_request = update_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
request_id=request_id,
|
||||
patch={
|
||||
@@ -393,8 +493,12 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
||||
"result": result.as_dict(),
|
||||
},
|
||||
)
|
||||
_emit_installer_daemon_notification(
|
||||
updated_request,
|
||||
event_kind="module_installer.request.completed" if result.return_code == 0 else "module_installer.request.failed",
|
||||
)
|
||||
except Exception as exc:
|
||||
update_module_installer_request(
|
||||
updated_request = update_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
request_id=request_id,
|
||||
patch={
|
||||
@@ -403,6 +507,34 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
_emit_installer_daemon_notification(updated_request, event_kind="module_installer.request.failed")
|
||||
|
||||
|
||||
def _emit_installer_daemon_notification(request: dict[str, object], *, event_kind: str) -> None:
|
||||
tenant_id = str(request.get("tenant_id") or "")
|
||||
if not tenant_id:
|
||||
return
|
||||
registry = build_runtime_notification_registry(settings)
|
||||
if registry is None:
|
||||
return
|
||||
status_value = str(request.get("status") or event_kind.rsplit(".", 1)[-1])
|
||||
try:
|
||||
with get_database().SessionLocal() as session:
|
||||
emitted = emit_module_installer_notification(
|
||||
session=session,
|
||||
registry=registry,
|
||||
tenant_id=tenant_id,
|
||||
request=request,
|
||||
event_kind=event_kind,
|
||||
subject=installer_notification_subject(event_kind, request),
|
||||
body_text=installer_notification_body(event_kind, request),
|
||||
recipient_id=str(request.get("requested_by") or "") or None,
|
||||
priority=installer_notification_priority(status_value),
|
||||
)
|
||||
if emitted:
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001 - notification bridge must not block installer work.
|
||||
return
|
||||
|
||||
|
||||
def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]:
|
||||
@@ -427,6 +559,7 @@ def _request_context(request: dict[str, object]) -> dict[str, object]:
|
||||
context: dict[str, object] = {
|
||||
"request_id": request.get("request_id"),
|
||||
"requested_by": request.get("requested_by"),
|
||||
"tenant_id": request.get("tenant_id"),
|
||||
}
|
||||
if request.get("retry_of"):
|
||||
context["retry_of"] = request["retry_of"]
|
||||
|
||||
98
src/govoplan_core/core/calendar.py
Normal file
98
src/govoplan_core/core/calendar.py
Normal file
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
CAPABILITY_CALENDAR_SCHEDULING = "calendar.scheduling"
|
||||
CAPABILITY_CALENDAR_OUTBOX = "calendar.outbox"
|
||||
CALENDAR_AVAILABILITY_READ_SCOPE = "calendar:availability:read"
|
||||
CALENDAR_EVENT_WRITE_SCOPE = "calendar:event:write"
|
||||
|
||||
|
||||
class CalendarCapabilityError(ValueError):
|
||||
"""Stable error raised by Calendar capability implementations."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CalendarEventRequest:
|
||||
summary: str
|
||||
start_at: datetime
|
||||
calendar_id: str | None = None
|
||||
description: str | None = None
|
||||
location: str | None = None
|
||||
status: str = "CONFIRMED"
|
||||
transparency: str = "OPAQUE"
|
||||
classification: str = "PUBLIC"
|
||||
end_at: datetime | None = None
|
||||
timezone: str | None = None
|
||||
attendees: tuple[Mapping[str, object], ...] = ()
|
||||
categories: tuple[str, ...] = ()
|
||||
related_to: tuple[Mapping[str, object], ...] = ()
|
||||
metadata: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CalendarEventRef:
|
||||
id: str
|
||||
calendar_id: str
|
||||
uid: str
|
||||
external_state: str = "local"
|
||||
outbox_operation_id: str | None = None
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CalendarSchedulingProvider(Protocol):
|
||||
def list_freebusy(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
calendar_ids: Sequence[str] | None = None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
...
|
||||
|
||||
def create_event(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str | None,
|
||||
request: CalendarEventRequest,
|
||||
) -> CalendarEventRef:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CalendarOutboxProvider(Protocol):
|
||||
"""Stable worker boundary for Calendar-owned external side effects."""
|
||||
|
||||
def dispatch_due(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str | None = None,
|
||||
limit: int = 50,
|
||||
) -> Mapping[str, object]:
|
||||
...
|
||||
|
||||
def calendar_scheduling_provider(registry: object | None) -> CalendarSchedulingProvider | None:
|
||||
if registry is None or not hasattr(registry, "has_capability"):
|
||||
return None
|
||||
if not registry.has_capability(CAPABILITY_CALENDAR_SCHEDULING):
|
||||
return None
|
||||
capability = registry.capability(CAPABILITY_CALENDAR_SCHEDULING)
|
||||
return capability if isinstance(capability, CalendarSchedulingProvider) else None
|
||||
|
||||
|
||||
def calendar_outbox_provider(registry: object | None) -> CalendarOutboxProvider | None:
|
||||
if registry is None or not hasattr(registry, "has_capability"):
|
||||
return None
|
||||
if not registry.has_capability(CAPABILITY_CALENDAR_OUTBOX):
|
||||
return None
|
||||
capability = registry.capability(CAPABILITY_CALENDAR_OUTBOX)
|
||||
return capability if isinstance(capability, CalendarOutboxProvider) else None
|
||||
@@ -12,6 +12,7 @@ from govoplan_core.admin.settings import get_system_settings
|
||||
from govoplan_core.core.change_sequence import record_change
|
||||
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
|
||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||
from govoplan_core.security.redaction import redact_secret_values
|
||||
from govoplan_core.security.time import utc_now
|
||||
|
||||
|
||||
@@ -377,17 +378,7 @@ def _sanitize_value(key: str, value: object) -> object:
|
||||
|
||||
|
||||
def _redact_secrets(value: object) -> object:
|
||||
if isinstance(value, dict):
|
||||
result: dict[str, object] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
|
||||
result[str(key)] = "<redacted>"
|
||||
else:
|
||||
result[str(key)] = _redact_secrets(item)
|
||||
return result
|
||||
if isinstance(value, list):
|
||||
return [_redact_secrets(item) for item in value]
|
||||
return value
|
||||
return redact_secret_values(value)
|
||||
|
||||
|
||||
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -8,8 +8,6 @@ from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Literal, Protocol, runtime_checkable
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from govoplan_core.core.module_package_catalog import (
|
||||
_canonical_catalog_bytes,
|
||||
@@ -23,6 +21,7 @@ from govoplan_core.core.module_package_catalog import (
|
||||
_load_private_key,
|
||||
_parse_trusted_keys,
|
||||
)
|
||||
from govoplan_core.security.http_fetch import fetch_http_text
|
||||
|
||||
|
||||
CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider"
|
||||
@@ -31,6 +30,20 @@ DiagnosticSeverity = Literal["blocker", "warning", "info"]
|
||||
PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ConfigurationCatalogValidationState:
|
||||
packages: tuple[dict[str, object], ...]
|
||||
channel: str | None
|
||||
sequence: int | None
|
||||
generated_at: str | None
|
||||
not_before: str | None
|
||||
expires_at: str | None
|
||||
signature_state: dict[str, object]
|
||||
freshness: dict[str, object]
|
||||
replay: dict[str, object]
|
||||
read_state: dict[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConfigurationModuleRequirement:
|
||||
module_id: str
|
||||
@@ -461,55 +474,148 @@ def validate_configuration_package_catalog(
|
||||
configured = source is not None and _catalog_source_exists(source)
|
||||
if source is not None and not _catalog_source_exists(source):
|
||||
return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}")
|
||||
read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() else None}
|
||||
read_state = _configuration_catalog_default_read_state()
|
||||
try:
|
||||
payload, read_state = _read_catalog_payload_with_metadata(source)
|
||||
packages = _normalize_catalog_packages(payload)
|
||||
channel = _catalog_channel(payload)
|
||||
sequence = _catalog_sequence(payload)
|
||||
generated_at = _catalog_optional_text(payload, "generated_at")
|
||||
not_before = _catalog_optional_text(payload, "not_before")
|
||||
expires_at = _catalog_optional_text(payload, "expires_at")
|
||||
effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted
|
||||
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
||||
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
||||
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
||||
freshness = _catalog_freshness_state(payload)
|
||||
replay = _catalog_replay_state(channel=channel, sequence=sequence)
|
||||
state = _configuration_catalog_validation_state(source, trusted_keys=effective_trusted_keys)
|
||||
read_state = state.read_state
|
||||
except Exception as exc:
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc))
|
||||
if signature_state.get("fatal"):
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"]))
|
||||
if effective_approved_channels and channel not in effective_approved_channels:
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=f"Configuration package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.")
|
||||
if effective_require_trusted and not signature_state["trusted"]:
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"] or "Configuration package catalog must be signed by a trusted key."))
|
||||
if not freshness["valid"]:
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(freshness["error"]))
|
||||
if not replay["valid"]:
|
||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(replay["error"]))
|
||||
warnings = [str(item) for item in freshness.get("warnings", ()) if item]
|
||||
warnings.extend(str(item) for item in replay.get("warnings", ()) if item)
|
||||
if not signature_state["signed"]:
|
||||
warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
||||
elif not signature_state["trusted"]:
|
||||
warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
||||
policy_error = _configuration_catalog_policy_error(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
require_trusted=effective_require_trusted,
|
||||
approved_channels=effective_approved_channels,
|
||||
)
|
||||
if policy_error is not None:
|
||||
return policy_error
|
||||
return _validation_result(
|
||||
source,
|
||||
valid=True,
|
||||
configured=configured,
|
||||
read_state=read_state,
|
||||
packages=packages,
|
||||
read_state=state.read_state,
|
||||
packages=state.packages,
|
||||
channel=state.channel,
|
||||
sequence=state.sequence,
|
||||
generated_at=state.generated_at,
|
||||
not_before=state.not_before,
|
||||
expires_at=state.expires_at,
|
||||
signature_state=state.signature_state,
|
||||
warnings=_configuration_catalog_warnings(state),
|
||||
)
|
||||
|
||||
|
||||
def _configuration_catalog_validation_state(
|
||||
source: Path | str | None,
|
||||
*,
|
||||
trusted_keys: dict[str, str],
|
||||
) -> _ConfigurationCatalogValidationState:
|
||||
payload, read_state = _read_catalog_payload_with_metadata(source)
|
||||
channel = _catalog_channel(payload)
|
||||
sequence = _catalog_sequence(payload)
|
||||
return _ConfigurationCatalogValidationState(
|
||||
packages=_normalize_catalog_packages(payload),
|
||||
channel=channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
not_before=not_before,
|
||||
expires_at=expires_at,
|
||||
signature_state=signature_state,
|
||||
warnings=warnings,
|
||||
generated_at=_catalog_optional_text(payload, "generated_at"),
|
||||
not_before=_catalog_optional_text(payload, "not_before"),
|
||||
expires_at=_catalog_optional_text(payload, "expires_at"),
|
||||
signature_state=_catalog_signature_state(payload, trusted_keys=trusted_keys),
|
||||
freshness=_catalog_freshness_state(payload),
|
||||
replay=_catalog_replay_state(channel=channel, sequence=sequence),
|
||||
read_state=read_state,
|
||||
)
|
||||
|
||||
|
||||
def _configuration_catalog_policy_error(
|
||||
source: Path | str | None,
|
||||
*,
|
||||
configured: bool,
|
||||
state: _ConfigurationCatalogValidationState,
|
||||
require_trusted: bool,
|
||||
approved_channels: tuple[str, ...],
|
||||
) -> dict[str, object] | None:
|
||||
if state.signature_state.get("fatal"):
|
||||
return _configuration_catalog_invalid_result(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
error=str(state.signature_state["error"]),
|
||||
)
|
||||
if approved_channels and state.channel not in approved_channels:
|
||||
return _configuration_catalog_invalid_result(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
error=(
|
||||
f"Configuration package catalog channel {state.channel!r} is not approved. "
|
||||
f"Approved channels: {', '.join(approved_channels)}."
|
||||
),
|
||||
)
|
||||
if require_trusted and not state.signature_state["trusted"]:
|
||||
return _configuration_catalog_invalid_result(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
error=str(state.signature_state["error"] or "Configuration package catalog must be signed by a trusted key."),
|
||||
)
|
||||
if not state.freshness["valid"]:
|
||||
return _configuration_catalog_invalid_result(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
error=str(state.freshness["error"]),
|
||||
)
|
||||
if not state.replay["valid"]:
|
||||
return _configuration_catalog_invalid_result(
|
||||
source,
|
||||
configured=configured,
|
||||
state=state,
|
||||
error=str(state.replay["error"]),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _configuration_catalog_invalid_result(
|
||||
source: Path | str | None,
|
||||
*,
|
||||
configured: bool,
|
||||
state: _ConfigurationCatalogValidationState,
|
||||
error: str,
|
||||
) -> dict[str, object]:
|
||||
return _validation_result(
|
||||
source,
|
||||
configured=configured,
|
||||
read_state=state.read_state,
|
||||
packages=(),
|
||||
channel=state.channel,
|
||||
sequence=state.sequence,
|
||||
generated_at=state.generated_at,
|
||||
not_before=state.not_before,
|
||||
expires_at=state.expires_at,
|
||||
signature_state=state.signature_state,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _configuration_catalog_warnings(state: _ConfigurationCatalogValidationState) -> list[str]:
|
||||
warnings = [str(item) for item in state.freshness.get("warnings", ()) if item]
|
||||
warnings.extend(str(item) for item in state.replay.get("warnings", ()) if item)
|
||||
if not state.signature_state["signed"]:
|
||||
warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
||||
elif not state.signature_state["trusted"]:
|
||||
warnings.append(str(state.signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
||||
return warnings
|
||||
|
||||
|
||||
def _configuration_catalog_default_read_state() -> dict[str, object]:
|
||||
cache_path = _configured_catalog_cache_path()
|
||||
return {"cache_used": False, "cache_path": str(cache_path) if cache_path else None}
|
||||
|
||||
|
||||
def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path:
|
||||
payload = _read_catalog_payload(path)
|
||||
if not isinstance(payload, dict):
|
||||
@@ -686,17 +792,14 @@ def _configured_trusted_keys_cache_path() -> Path | None:
|
||||
|
||||
|
||||
def _read_trusted_keys_url(url: str) -> str:
|
||||
if not _is_http_url(url):
|
||||
raise ValueError("Trusted configuration catalog key URL must use http:// or https://.")
|
||||
cache_path = _configured_trusted_keys_cache_path()
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
body = fetch_http_text(url, timeout=15, label="Trusted configuration catalog key URL")
|
||||
if cache_path is not None:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_path.write_text(body, encoding="utf-8")
|
||||
return body
|
||||
except (OSError, urllib.error.URLError):
|
||||
except OSError:
|
||||
if cache_path is not None and cache_path.exists():
|
||||
return cache_path.read_text(encoding="utf-8")
|
||||
raise
|
||||
@@ -714,13 +817,12 @@ def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[obje
|
||||
return {"packages": []}, metadata
|
||||
if isinstance(source, str) and _is_http_url(source):
|
||||
try:
|
||||
with urllib.request.urlopen(source, timeout=15) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
body = fetch_http_text(source, timeout=15, label="Configuration package catalog URL")
|
||||
if cache_path is not None:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_path.write_text(body, encoding="utf-8")
|
||||
return json.loads(body), metadata
|
||||
except (OSError, urllib.error.URLError):
|
||||
except OSError:
|
||||
if cache_path is not None and cache_path.exists():
|
||||
metadata["cache_used"] = True
|
||||
return json.loads(cache_path.read_text(encoding="utf-8")), metadata
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Literal
|
||||
from typing import Literal
|
||||
|
||||
from govoplan_core.security.permissions import scopes_grant
|
||||
from govoplan_core.security.redaction import contains_plain_secret
|
||||
|
||||
|
||||
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
|
||||
@@ -21,7 +21,7 @@ class ConfigurationFieldSafety:
|
||||
storage: str
|
||||
ui_managed: bool
|
||||
risk: ConfigurationRisk
|
||||
secret_handling: SecretHandling = "none"
|
||||
secret_handling: SecretHandling = "none" # noqa: S105 - policy vocabulary.
|
||||
required_scopes: tuple[str, ...] = ()
|
||||
dry_run_required: bool = False
|
||||
validation_required: bool = True
|
||||
@@ -68,7 +68,7 @@ class ConfigurationChangeSafetyPlan:
|
||||
maintenance_required: bool = False
|
||||
maintenance_satisfied: bool = False
|
||||
rollback_history_required: bool = False
|
||||
secret_handling: SecretHandling = "none"
|
||||
secret_handling: SecretHandling = "none" # noqa: S105 - policy vocabulary.
|
||||
audit_event: str | None = None
|
||||
policy_explanation: str | None = None
|
||||
blockers: tuple[str, ...] = ()
|
||||
@@ -96,6 +96,16 @@ class ConfigurationChangeSafetyPlan:
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ConfigurationChangeSafetyState:
|
||||
field: ConfigurationFieldSafety
|
||||
missing_scopes: tuple[str, ...]
|
||||
blockers: tuple[str, ...]
|
||||
warnings: tuple[str, ...]
|
||||
approval_required: bool
|
||||
approval_satisfied: bool
|
||||
|
||||
|
||||
_CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
ConfigurationFieldSafety(
|
||||
key="module_management.desired_enabled",
|
||||
@@ -229,7 +239,7 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="module_settings",
|
||||
ui_managed=True,
|
||||
risk="high",
|
||||
secret_handling="reference_only",
|
||||
secret_handling="reference_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
required_scopes=("mail_servers:manage_credentials",),
|
||||
validation_required=True,
|
||||
audit_event="mail_server_profile.credential_updated",
|
||||
@@ -245,14 +255,14 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="module_settings",
|
||||
ui_managed=True,
|
||||
risk="high",
|
||||
secret_handling="reference_only",
|
||||
secret_handling="reference_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
required_scopes=("files:file:admin",),
|
||||
dry_run_required=True,
|
||||
policy_explanation_required=True,
|
||||
audit_event="files.connector_profile.updated",
|
||||
two_person_approval_required=True,
|
||||
rollback_history_required=True,
|
||||
notes="Connector endpoints are UI-manageable, but passwords/tokens remain env or secret refs.",
|
||||
notes="Connector endpoints are UI-manageable. API-managed credentials are encrypted or use scoped secret refs; process-environment references remain deployment-owned.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="DATABASE_URL",
|
||||
@@ -262,7 +272,7 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="destructive",
|
||||
secret_handling="env_only",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Database connectivity remains deployment-managed and must not be changed from the running UI.",
|
||||
),
|
||||
@@ -274,11 +284,179 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="destructive",
|
||||
secret_handling="env_only",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
two_person_approval_required=True,
|
||||
notes="Encryption roots remain out of band; UI may only report missing/rotated state.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
|
||||
label="Private-network connector access",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="This deployment-wide egress boundary remains out of band and applies to every connector worker.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES",
|
||||
label="Structured connector response limit",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="A deployment-wide memory-safety limit applied consistently by API and worker processes.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES",
|
||||
label="Connector file-transfer limit",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="A deployment-wide hard ceiling; individual module upload policies may impose lower limits.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST",
|
||||
label="Connector secret environment allowlist",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="destructive",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Exact environment names available only to deployment-owned connector profiles; tenant/API profiles cannot select process variables.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST",
|
||||
label="Connector CA bundle allowlist",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Exact absolute CA bundle paths approved by the deployment and mounted consistently on every connector worker.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES",
|
||||
label="HTTP request-body limit",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="A deployment-wide hard ceiling; endpoint-specific upload policies may impose lower limits.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_HTTP_HSTS_SECONDS",
|
||||
label="HTTP Strict Transport Security duration",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Deployment-managed browser transport policy; use only after the public service is HTTPS-only.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="AUTH_LOGIN_THROTTLE_ENABLED",
|
||||
label="Interactive login throttling",
|
||||
owner_module="access",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Disabling the deployment login throttle weakens protection against password guessing.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT",
|
||||
label="Login failures per identity",
|
||||
owner_module="access",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Shared through Redis when available, with a bounded process-local fallback.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="AUTH_LOGIN_THROTTLE_CLIENT_LIMIT",
|
||||
label="Login failures per client",
|
||||
owner_module="access",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Counts the direct peer after the deployment's trusted-proxy boundary is applied.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="AUTH_LOGIN_THROTTLE_WINDOW_SECONDS",
|
||||
label="Login throttle window",
|
||||
owner_module="access",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Fixed counter window shared by identity and direct-client limits.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS",
|
||||
label="Login throttle Redis retry interval",
|
||||
owner_module="access",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="medium",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Controls how quickly a worker retries the distributed counter after falling back locally.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_TRUSTED_HOSTS",
|
||||
label="Trusted HTTP hosts",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Host-header validation is deployment-managed and must cover every public API host.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="FORWARDED_ALLOW_IPS",
|
||||
label="Trusted reverse proxy addresses",
|
||||
owner_module="core",
|
||||
scope="system",
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
maintenance_required=True,
|
||||
notes="Uvicorn trusts forwarded client and scheme data only from this deployment-managed boundary.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
key="GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS",
|
||||
label="Module package catalog trusted keys",
|
||||
@@ -287,7 +465,7 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
notes="Trust roots are deployment-managed; UI can validate catalogs but should not edit key material.",
|
||||
),
|
||||
ConfigurationFieldSafety(
|
||||
@@ -298,7 +476,7 @@ _CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||
storage="environment",
|
||||
ui_managed=False,
|
||||
risk="high",
|
||||
secret_handling="env_only",
|
||||
secret_handling="env_only", # noqa: S106 # nosec B106 - policy vocabulary.
|
||||
notes="Configuration package trust roots are deployment-managed.",
|
||||
),
|
||||
)
|
||||
@@ -338,23 +516,56 @@ def plan_configuration_change(
|
||||
) -> ConfigurationChangeSafetyPlan:
|
||||
field = classify_configuration_field(key)
|
||||
if field is None:
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=key,
|
||||
allowed=False,
|
||||
field=None,
|
||||
blockers=("unknown_configuration_field",),
|
||||
policy_explanation="This setting is not in the configuration safety catalog.",
|
||||
)
|
||||
return _unknown_configuration_plan(key)
|
||||
if not include_env_only and not field.ui_managed:
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=key,
|
||||
allowed=False,
|
||||
field=field,
|
||||
risk=field.risk,
|
||||
secret_handling=field.secret_handling,
|
||||
blockers=("deployment_managed",),
|
||||
policy_explanation="This setting is deployment-managed and cannot be edited through the UI.",
|
||||
)
|
||||
return _deployment_managed_configuration_plan(key, field)
|
||||
state = _configuration_change_safety_state(
|
||||
field,
|
||||
actor_scopes=actor_scopes,
|
||||
value=value,
|
||||
dry_run=dry_run,
|
||||
maintenance_mode=maintenance_mode,
|
||||
approval_count=approval_count,
|
||||
)
|
||||
return _configuration_change_safety_plan(
|
||||
field,
|
||||
state=state,
|
||||
dry_run=dry_run,
|
||||
maintenance_mode=maintenance_mode,
|
||||
)
|
||||
|
||||
|
||||
def _unknown_configuration_plan(key: str) -> ConfigurationChangeSafetyPlan:
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=key,
|
||||
allowed=False,
|
||||
field=None,
|
||||
blockers=("unknown_configuration_field",),
|
||||
policy_explanation="This setting is not in the configuration safety catalog.",
|
||||
)
|
||||
|
||||
|
||||
def _deployment_managed_configuration_plan(key: str, field: ConfigurationFieldSafety) -> ConfigurationChangeSafetyPlan:
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=key,
|
||||
allowed=False,
|
||||
field=field,
|
||||
risk=field.risk,
|
||||
secret_handling=field.secret_handling,
|
||||
blockers=("deployment_managed",),
|
||||
policy_explanation="This setting is deployment-managed and cannot be edited through the UI.",
|
||||
)
|
||||
|
||||
|
||||
def _configuration_change_safety_state(
|
||||
field: ConfigurationFieldSafety,
|
||||
*,
|
||||
actor_scopes: tuple[str, ...] | list[str],
|
||||
value: object,
|
||||
dry_run: bool,
|
||||
maintenance_mode: bool,
|
||||
approval_count: int,
|
||||
) -> _ConfigurationChangeSafetyState:
|
||||
blockers: list[str] = []
|
||||
warnings: list[str] = []
|
||||
missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope))
|
||||
@@ -370,30 +581,47 @@ def plan_configuration_change(
|
||||
approval_satisfied = not approval_required or approval_count >= 2
|
||||
if approval_required and not approval_satisfied:
|
||||
blockers.append("two_person_approval_required")
|
||||
if field.secret_handling == "reference_only" and _contains_plain_secret(value):
|
||||
if field.secret_handling == "reference_only" and _contains_plain_secret(value): # noqa: S105 # nosec B105 - policy vocabulary.
|
||||
blockers.append("secret_reference_required")
|
||||
if field.secret_handling == "env_only" and value is not None:
|
||||
if field.secret_handling == "env_only" and value is not None: # noqa: S105 # nosec B105 - policy vocabulary.
|
||||
blockers.append("env_only_secret")
|
||||
if field.rollback_history_required:
|
||||
warnings.append("rollback_history_required")
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=field.key,
|
||||
allowed=not blockers,
|
||||
return _ConfigurationChangeSafetyState(
|
||||
field=field,
|
||||
risk=field.risk,
|
||||
missing_scopes=missing_scopes,
|
||||
dry_run_required=field.dry_run_required,
|
||||
dry_run_satisfied=not field.dry_run_required or dry_run,
|
||||
blockers=tuple(dict.fromkeys(blockers)),
|
||||
warnings=tuple(dict.fromkeys(warnings)),
|
||||
approval_required=approval_required,
|
||||
approval_satisfied=approval_satisfied,
|
||||
)
|
||||
|
||||
|
||||
def _configuration_change_safety_plan(
|
||||
field: ConfigurationFieldSafety,
|
||||
*,
|
||||
state: _ConfigurationChangeSafetyState,
|
||||
dry_run: bool,
|
||||
maintenance_mode: bool,
|
||||
) -> ConfigurationChangeSafetyPlan:
|
||||
return ConfigurationChangeSafetyPlan(
|
||||
key=field.key,
|
||||
allowed=not state.blockers,
|
||||
field=field,
|
||||
risk=field.risk,
|
||||
missing_scopes=state.missing_scopes,
|
||||
dry_run_required=field.dry_run_required,
|
||||
dry_run_satisfied=not field.dry_run_required or dry_run,
|
||||
approval_required=state.approval_required,
|
||||
approval_satisfied=state.approval_satisfied,
|
||||
maintenance_required=field.maintenance_required,
|
||||
maintenance_satisfied=not field.maintenance_required or maintenance_mode,
|
||||
rollback_history_required=field.rollback_history_required,
|
||||
secret_handling=field.secret_handling,
|
||||
audit_event=field.audit_event,
|
||||
policy_explanation=_policy_explanation(field),
|
||||
blockers=tuple(dict.fromkeys(blockers)),
|
||||
warnings=tuple(dict.fromkeys(warnings)),
|
||||
blockers=state.blockers,
|
||||
warnings=state.warnings,
|
||||
)
|
||||
|
||||
|
||||
@@ -405,19 +633,10 @@ def _policy_explanation(field: ConfigurationFieldSafety) -> str:
|
||||
parts.append("requires two-person approval")
|
||||
if field.maintenance_required:
|
||||
parts.append("requires maintenance mode")
|
||||
if field.secret_handling != "none":
|
||||
if field.secret_handling != "none": # noqa: S105 # nosec B105 - policy vocabulary.
|
||||
parts.append(f"uses {field.secret_handling} secret handling")
|
||||
return "; ".join(parts) + "."
|
||||
|
||||
|
||||
def _contains_plain_secret(value: object) -> bool:
|
||||
if not isinstance(value, Mapping):
|
||||
return False
|
||||
secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
|
||||
for key, item in value.items():
|
||||
name = str(key).strip().casefold()
|
||||
if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
|
||||
return True
|
||||
if isinstance(item, Mapping) and _contains_plain_secret(item):
|
||||
return True
|
||||
return False
|
||||
return contains_plain_secret(value)
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Literal, Protocol, runtime_checkable
|
||||
|
||||
IDENTITY_MODULE_ID = "identity"
|
||||
CAPABILITY_IDENTITY_DIRECTORY = "identity.directory"
|
||||
CAPABILITY_IDENTITY_SEARCH = "identity.search"
|
||||
|
||||
IdentityStatus = Literal["active", "inactive", "suspended"]
|
||||
|
||||
@@ -44,3 +45,15 @@ class IdentityDirectory(Protocol):
|
||||
|
||||
def accounts_for_identity(self, identity_id: str) -> Sequence[IdentityAccountLinkRef]:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class IdentitySearchProvider(Protocol):
|
||||
def search_identities(
|
||||
self,
|
||||
query: str | None = None,
|
||||
*,
|
||||
include_inactive: bool = False,
|
||||
limit: int = 25,
|
||||
) -> Sequence[IdentityRef]:
|
||||
...
|
||||
|
||||
@@ -3,9 +3,11 @@ from __future__ import annotations
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
from sqlalchemy.engine import make_url
|
||||
@@ -72,6 +74,25 @@ class ConfigValidationResult:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RuntimeProfile:
|
||||
name: str
|
||||
production: bool
|
||||
production_like: bool
|
||||
local: bool
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _ConfigIssueCollector:
|
||||
strict: bool
|
||||
issues: list[ConfigIssue]
|
||||
|
||||
def add(self, level: ConfigIssueLevel, key: str, message: str, action: str) -> None:
|
||||
if self.strict and level == "warning":
|
||||
level = "error"
|
||||
self.issues.append(ConfigIssue(level=level, key=key, message=message, action=action))
|
||||
|
||||
|
||||
_LOCAL_PROFILES = {"dev", "local", "local-dev", "test"}
|
||||
_PRODUCTION_PROFILES = {"prod", "production", "self-hosted"}
|
||||
_PRODUCTION_LIKE_PROFILES = {"staging", "production-like", "production-like-dev"}
|
||||
@@ -114,95 +135,192 @@ def validate_runtime_configuration(
|
||||
strict: bool = False,
|
||||
) -> ConfigValidationResult:
|
||||
env = dict(os.environ if environ is None else environ)
|
||||
clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV"))
|
||||
issues: list[ConfigIssue] = []
|
||||
runtime = _runtime_profile(env, profile=profile)
|
||||
collector = _ConfigIssueCollector(strict=strict, issues=[])
|
||||
_validate_app_env(env, runtime, collector)
|
||||
_validate_database_settings(env, runtime, collector)
|
||||
_validate_master_key(env, runtime, collector)
|
||||
_validate_enabled_modules(env, runtime, collector)
|
||||
_validate_async_and_auth_settings(env, runtime, collector)
|
||||
_validate_cors_settings(env, runtime, collector)
|
||||
_validate_file_storage_settings(env, runtime, collector)
|
||||
_validate_outbound_connector_policy(env, runtime, collector)
|
||||
_validate_module_catalog_trust(env, runtime, collector)
|
||||
return ConfigValidationResult(profile=runtime.name, issues=tuple(collector.issues))
|
||||
|
||||
|
||||
def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _RuntimeProfile:
|
||||
clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV"))
|
||||
production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"}
|
||||
production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES
|
||||
local = clean_profile in _LOCAL_PROFILES and not production_like
|
||||
return _RuntimeProfile(
|
||||
name=clean_profile,
|
||||
production=production,
|
||||
production_like=production_like,
|
||||
local=clean_profile in _LOCAL_PROFILES and not production_like,
|
||||
)
|
||||
|
||||
def issue(level: ConfigIssueLevel, key: str, message: str, action: str) -> None:
|
||||
if strict and level == "warning":
|
||||
level = "error"
|
||||
issues.append(ConfigIssue(level=level, key=key, message=message, action=action))
|
||||
|
||||
def _validate_app_env(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
app_env = _clean(env.get("APP_ENV"))
|
||||
if not app_env and production_like:
|
||||
issue("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.")
|
||||
elif app_env.lower() in {"dev", "test", "local"} and production_like:
|
||||
issue("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {clean_profile!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.")
|
||||
if not app_env and runtime.production_like:
|
||||
collector.add("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.")
|
||||
elif app_env.lower() in {"dev", "test", "local"} and runtime.production_like:
|
||||
collector.add("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.")
|
||||
|
||||
|
||||
def _validate_database_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
database_url = _clean(env.get("DATABASE_URL"))
|
||||
if not database_url:
|
||||
issue("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.")
|
||||
else:
|
||||
backend = _database_backend(database_url)
|
||||
if backend is None:
|
||||
issue("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.")
|
||||
elif backend == "sqlite" and production_like:
|
||||
issue("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.")
|
||||
elif backend != "postgresql" and production:
|
||||
issue("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.")
|
||||
if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")):
|
||||
issue("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.")
|
||||
collector.add("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.")
|
||||
return
|
||||
backend = _database_backend(database_url)
|
||||
if backend is None:
|
||||
collector.add("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.")
|
||||
return
|
||||
if backend == "sqlite" and runtime.production_like:
|
||||
collector.add("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.")
|
||||
elif backend != "postgresql" and runtime.production:
|
||||
collector.add("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.")
|
||||
if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")):
|
||||
collector.add("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.")
|
||||
|
||||
|
||||
def _validate_master_key(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
master_key = _clean(env.get("MASTER_KEY_B64"))
|
||||
if not master_key and not local:
|
||||
issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.")
|
||||
elif master_key:
|
||||
error = _master_key_error(master_key)
|
||||
if error:
|
||||
issue("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.")
|
||||
elif "change-me" in master_key.lower() or "generate" in master_key.lower():
|
||||
issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.")
|
||||
if not master_key and not runtime.local:
|
||||
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.")
|
||||
return
|
||||
if not master_key:
|
||||
return
|
||||
error = _master_key_error(master_key)
|
||||
if error:
|
||||
collector.add("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.")
|
||||
elif "change-me" in master_key.lower() or "generate" in master_key.lower():
|
||||
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.")
|
||||
|
||||
|
||||
def _validate_enabled_modules(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
enabled_modules = _csv(env.get("ENABLED_MODULES"))
|
||||
if not enabled_modules and production_like:
|
||||
issue("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.")
|
||||
elif "access" not in enabled_modules and production_like:
|
||||
issue("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.")
|
||||
elif enabled_modules and "admin" not in enabled_modules and production_like:
|
||||
issue("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.")
|
||||
if not enabled_modules and runtime.production_like:
|
||||
collector.add("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.")
|
||||
elif "access" not in enabled_modules and runtime.production_like:
|
||||
collector.add("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.")
|
||||
elif enabled_modules and "admin" not in enabled_modules and runtime.production_like:
|
||||
collector.add("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.")
|
||||
|
||||
celery_enabled = _truthy(env.get("CELERY_ENABLED"))
|
||||
if celery_enabled and not _clean(env.get("REDIS_URL")):
|
||||
issue("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.")
|
||||
|
||||
if production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")):
|
||||
issue("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.")
|
||||
def _validate_async_and_auth_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
if _truthy(env.get("CELERY_ENABLED")) and not _clean(env.get("REDIS_URL")):
|
||||
collector.add("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.")
|
||||
if runtime.production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")):
|
||||
collector.add("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.")
|
||||
if runtime.production and not _truthy(env.get("AUTH_COOKIE_SECURE")):
|
||||
collector.add("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.")
|
||||
|
||||
if production and not _truthy(env.get("AUTH_COOKIE_SECURE")):
|
||||
issue("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.")
|
||||
|
||||
def _validate_cors_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
cors_origins = _csv(env.get("CORS_ORIGINS"))
|
||||
if production_like and not cors_origins:
|
||||
issue("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.")
|
||||
elif "*" in cors_origins and production_like:
|
||||
issue("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.")
|
||||
elif production and set(cors_origins) <= _DEFAULT_LOCAL_CORS:
|
||||
issue("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.")
|
||||
if runtime.production_like and not cors_origins:
|
||||
collector.add("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.")
|
||||
elif "*" in cors_origins and runtime.production_like:
|
||||
collector.add("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.")
|
||||
elif runtime.production and set(cors_origins) <= _DEFAULT_LOCAL_CORS:
|
||||
collector.add("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.")
|
||||
trusted_hosts = _csv(env.get("GOVOPLAN_TRUSTED_HOSTS"))
|
||||
if runtime.production_like and not trusted_hosts:
|
||||
collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Trusted HTTP hosts are not configured.", "Set GOVOPLAN_TRUSTED_HOSTS to the exact API host names accepted by this deployment.")
|
||||
elif "*" in trusted_hosts and runtime.production_like:
|
||||
collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Wildcard trusted hosts are not allowed for production-like installs.", "Replace `*` with exact host names or narrowly scoped `*.example.org` entries.")
|
||||
forwarded_allow_ips = _csv(env.get("FORWARDED_ALLOW_IPS"))
|
||||
if runtime.production_like and "*" in forwarded_allow_ips:
|
||||
collector.add("error", "FORWARDED_ALLOW_IPS", "Proxy headers must not be trusted from every address.", "Set FORWARDED_ALLOW_IPS to the reverse proxy address or network passed to Uvicorn.")
|
||||
|
||||
|
||||
def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local"
|
||||
if storage_backend == "local":
|
||||
if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and production_like:
|
||||
issue("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.")
|
||||
elif production:
|
||||
issue("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.")
|
||||
_validate_local_file_storage(env, runtime, collector)
|
||||
elif storage_backend == "s3":
|
||||
for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"):
|
||||
if not _clean(env.get(key)):
|
||||
issue("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.")
|
||||
_validate_s3_file_storage(env, collector)
|
||||
else:
|
||||
issue("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.")
|
||||
collector.add("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.")
|
||||
|
||||
|
||||
def _validate_local_file_storage(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and runtime.production_like:
|
||||
collector.add("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.")
|
||||
elif runtime.production:
|
||||
collector.add("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.")
|
||||
|
||||
|
||||
def _validate_s3_file_storage(env: Mapping[str, str], collector: _ConfigIssueCollector) -> None:
|
||||
for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"):
|
||||
if not _clean(env.get(key)):
|
||||
collector.add("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.")
|
||||
|
||||
|
||||
def _validate_outbound_connector_policy(
|
||||
env: Mapping[str, str],
|
||||
runtime: _RuntimeProfile,
|
||||
collector: _ConfigIssueCollector,
|
||||
) -> None:
|
||||
private_networks = _clean(env.get("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS")).lower()
|
||||
if runtime.production_like and private_networks not in {"true", "false", "1", "0", "yes", "no", "on", "off"}:
|
||||
collector.add(
|
||||
"error",
|
||||
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
|
||||
"Private-network connector access must be an explicit deployment decision.",
|
||||
"Set GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=false for public-only egress, or true when this deployment must reach internal services.",
|
||||
)
|
||||
for key in (
|
||||
"GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES",
|
||||
"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES",
|
||||
"GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES",
|
||||
):
|
||||
value = _clean(env.get(key))
|
||||
if not value:
|
||||
continue
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError:
|
||||
parsed = 0
|
||||
if parsed <= 0:
|
||||
collector.add("error", key, f"{key} must be a positive byte count.", "Use a positive integer byte limit.")
|
||||
secret_env_names = [
|
||||
item.strip()
|
||||
for item in env.get("GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", "").split(",")
|
||||
if item.strip()
|
||||
]
|
||||
if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", item) is None for item in secret_env_names):
|
||||
collector.add(
|
||||
"error",
|
||||
"GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST",
|
||||
"Connector secret environment allowlist contains an invalid variable name.",
|
||||
"Use a comma-separated list of exact environment variable names, or leave the setting empty.",
|
||||
)
|
||||
ca_bundle_paths = [
|
||||
item.strip()
|
||||
for item in env.get("GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST", "").split(",")
|
||||
if item.strip()
|
||||
]
|
||||
if any(not Path(item).is_absolute() for item in ca_bundle_paths):
|
||||
collector.add(
|
||||
"error",
|
||||
"GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST",
|
||||
"Connector CA bundle allowlist contains a non-absolute path.",
|
||||
"Use comma-separated absolute paths mounted on every API and connector worker, or leave the setting empty.",
|
||||
)
|
||||
|
||||
|
||||
def _validate_module_catalog_trust(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||
catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG"))
|
||||
if production and catalog_source:
|
||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")):
|
||||
issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.")
|
||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")):
|
||||
issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.")
|
||||
|
||||
return ConfigValidationResult(profile=clean_profile, issues=tuple(issues))
|
||||
if not runtime.production or not catalog_source:
|
||||
return
|
||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")):
|
||||
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.")
|
||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")):
|
||||
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.")
|
||||
|
||||
|
||||
def _self_hosted_env_template(master_key: str) -> str:
|
||||
@@ -220,9 +338,32 @@ ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,aud
|
||||
|
||||
CELERY_ENABLED=true
|
||||
REDIS_URL=redis://127.0.0.1:6379/0
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||
SCHEDULING_CANCELLATION_NOTICE_DAYS=30
|
||||
|
||||
# Deployment-wide connector egress policy. Enable private networks only when
|
||||
# this installation intentionally integrates with internal services.
|
||||
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=false
|
||||
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||
# Exact names/paths only. Keep empty until a deployment-owned connector needs them.
|
||||
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||
GOVOPLAN_HTTP_HSTS_SECONDS=31536000
|
||||
|
||||
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||
|
||||
CORS_ORIGINS=https://govoplan.example.org
|
||||
GOVOPLAN_TRUSTED_HOSTS=govoplan.example.org
|
||||
# Uvicorn reads FORWARDED_ALLOW_IPS when proxy headers are enabled. Keep this
|
||||
# restricted to the actual reverse proxy address or network.
|
||||
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||
AUTH_COOKIE_SECURE=true
|
||||
AUTH_COOKIE_SAMESITE=lax
|
||||
AUTH_COOKIE_DOMAIN=
|
||||
@@ -263,10 +404,28 @@ DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
REDIS_URL=redis://127.0.0.1:56379/0
|
||||
CELERY_ENABLED=true
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
CELERY_QUEUES=send_email,append_sent,notifications,calendar,default
|
||||
CALENDAR_OUTBOX_TERMINAL_RETENTION_DAYS=90
|
||||
SCHEDULING_CANCELLATION_NOTICE_DAYS=30
|
||||
|
||||
GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS=true
|
||||
GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES=16777216
|
||||
GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES=536870912
|
||||
GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST=
|
||||
GOVOPLAN_CONNECTOR_CA_BUNDLE_ALLOWLIST=
|
||||
GOVOPLAN_HTTP_MAX_REQUEST_BODY_BYTES=536870912
|
||||
GOVOPLAN_HTTP_HSTS_SECONDS=0
|
||||
|
||||
AUTH_LOGIN_THROTTLE_ENABLED=true
|
||||
AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10
|
||||
AUTH_LOGIN_THROTTLE_CLIENT_LIMIT=100
|
||||
AUTH_LOGIN_THROTTLE_WINDOW_SECONDS=900
|
||||
AUTH_LOGIN_THROTTLE_REDIS_RETRY_SECONDS=30
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||
GOVOPLAN_TRUSTED_HOSTS=127.0.0.1,localhost,testserver
|
||||
FORWARDED_ALLOW_IPS=127.0.0.1
|
||||
AUTH_COOKIE_SECURE=false
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||
@@ -299,8 +458,8 @@ def _master_key_error(value: str) -> str | None:
|
||||
try:
|
||||
Fernet(candidate)
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
except (TypeError, ValueError):
|
||||
raw = None
|
||||
try:
|
||||
raw = base64.b64decode(candidate)
|
||||
except Exception:
|
||||
@@ -309,6 +468,6 @@ def _master_key_error(value: str) -> str | None:
|
||||
return "MASTER_KEY_B64 must decode to exactly 32 bytes."
|
||||
try:
|
||||
Fernet(base64.urlsafe_b64encode(raw))
|
||||
except Exception:
|
||||
except (TypeError, ValueError):
|
||||
return "MASTER_KEY_B64 is not usable as a Fernet key."
|
||||
return None
|
||||
|
||||
@@ -79,15 +79,19 @@ def drop_table_retirement_provider(
|
||||
warnings.append("Tables not present and therefore skipped: " + ", ".join(missing_names))
|
||||
|
||||
def executor(execute_session: object, _module_id: str) -> None:
|
||||
if not hasattr(execute_session, "get_bind"):
|
||||
if not hasattr(execute_session, "connection"):
|
||||
raise RuntimeError("No database session is available for destructive table retirement.")
|
||||
execute_bind = execute_session.get_bind() # type: ignore[attr-defined]
|
||||
live_inspector = inspect(execute_bind)
|
||||
# Enlist schema retirement in the caller's active transaction. An
|
||||
# Engine returned by Session.get_bind() may acquire a second
|
||||
# connection, separating the DROP from module-specific secret
|
||||
# scrubbing/audit writes and deadlocking on their uncommitted locks.
|
||||
execute_connection = execute_session.connection() # type: ignore[attr-defined]
|
||||
live_inspector = inspect(execute_connection)
|
||||
live_tables = [table for table in tables if live_inspector.has_table(table.name)]
|
||||
if not live_tables:
|
||||
return
|
||||
metadata = live_tables[0].metadata
|
||||
metadata.drop_all(bind=execute_bind, tables=live_tables, checkfirst=True)
|
||||
metadata.drop_all(bind=execute_connection, tables=live_tables, checkfirst=True)
|
||||
|
||||
return MigrationRetirementPlan(
|
||||
supported=True,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user