Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9a0c467d55 | |||
| a00ef54821 | |||
| edb4687826 | |||
| fcfe0b69a3 | |||
| 715bdcbebe | |||
| 060f4da751 | |||
| c32951393a | |||
| 7b85d6deae | |||
| 2d2d9e7bc7 | |||
| dc1a250797 | |||
| 7b7cc8ada7 | |||
| 722c9e5d1c | |||
| 63e54a67be | |||
| 12b623bec9 | |||
| b788afcae1 | |||
| 2b0cdf13f3 |
28
.dockerignore
Normal file
28
.dockerignore
Normal file
@@ -0,0 +1,28 @@
|
||||
.git
|
||||
.venv
|
||||
node_modules
|
||||
webui/node_modules
|
||||
audit-reports
|
||||
runtime
|
||||
dist
|
||||
build
|
||||
coverage
|
||||
htmlcov
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.log
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.cache
|
||||
.component-test-build
|
||||
.module-test-build
|
||||
.policy-test-build
|
||||
.template-preview-test-build
|
||||
.import-test-build
|
||||
webui/.component-test-build
|
||||
webui/.module-test-build
|
||||
webui/.policy-test-build
|
||||
webui/.template-preview-test-build
|
||||
webui/.import-test-build
|
||||
32
.env.example
32
.env.example
@@ -1,32 +0,0 @@
|
||||
# GovOPlaN self-hosted install configuration.
|
||||
# Copy to a deployment-local .env or secret store. Do not commit populated secrets.
|
||||
|
||||
APP_ENV=production
|
||||
GOVOPLAN_INSTALL_PROFILE=self-hosted
|
||||
MASTER_KEY_B64=<generate-with-govoplan-config-env-template-generate-secrets>
|
||||
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,docs,ops
|
||||
|
||||
CELERY_ENABLED=true
|
||||
REDIS_URL=redis://127.0.0.1:6379/0
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
|
||||
CORS_ORIGINS=https://govoplan.example.org
|
||||
AUTH_COOKIE_SECURE=true
|
||||
AUTH_COOKIE_SAMESITE=lax
|
||||
AUTH_COOKIE_DOMAIN=
|
||||
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
|
||||
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
|
||||
|
||||
DEV_AUTO_MIGRATE_ENABLED=false
|
||||
DEV_BOOTSTRAP_ENABLED=false
|
||||
DEV_MAILBOX_API_ENABLED=false
|
||||
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable
|
||||
@@ -1,50 +0,0 @@
|
||||
name: Dependency Audit
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
schedule:
|
||||
- cron: "23 3 * * 1"
|
||||
|
||||
jobs:
|
||||
dependency-audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Configure SSH for release dependencies
|
||||
env:
|
||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||
chmod 600 ~/.ssh/known_hosts
|
||||
- name: Install backend dev audit dependencies
|
||||
run: |
|
||||
python -m venv .venv
|
||||
.venv/bin/python -m pip install --upgrade pip
|
||||
.venv/bin/python -m pip install -r requirements-release.txt
|
||||
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
|
||||
- name: Install WebUI release dependencies
|
||||
working-directory: webui
|
||||
run: |
|
||||
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); for (const key of ['dependencies','devDependencies','peerDependencies','optionalDependencies','overrides']) if (rel[key]) pkg[key]=rel[key]; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
||||
rm -f package-lock.json
|
||||
npm cache clean --force
|
||||
npm install --prefer-online
|
||||
- name: Run dependency audits
|
||||
run: bash scripts/check-dependency-audits.sh
|
||||
@@ -1,48 +0,0 @@
|
||||
name: Module Matrix
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
module-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Configure SSH for release dependencies
|
||||
env:
|
||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
||||
chmod 600 ~/.ssh/known_hosts
|
||||
- name: Install backend release dependencies
|
||||
run: |
|
||||
python -m venv .venv
|
||||
.venv/bin/python -m pip install --upgrade pip
|
||||
.venv/bin/python -m pip install -r requirements-release.txt
|
||||
.venv/bin/python -m pip install '.[dev]'
|
||||
- name: Install WebUI release dependencies with test scripts
|
||||
working-directory: webui
|
||||
run: |
|
||||
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); for (const key of ['dependencies','devDependencies','peerDependencies','optionalDependencies','overrides']) if (rel[key]) pkg[key]=rel[key]; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
||||
rm -f package-lock.json
|
||||
npm cache clean --force
|
||||
npm install --prefer-online
|
||||
- name: Run module matrix and contract tests
|
||||
run: bash scripts/check-module-matrix.sh
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -148,6 +148,9 @@ webui/.policy-test-build/
|
||||
webui/.template-preview-test-build/
|
||||
webui/.import-test-build/
|
||||
|
||||
# Security audit reports
|
||||
audit-reports/
|
||||
|
||||
# ---> Python
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
10
AGENTS.md
10
AGENTS.md
@@ -19,9 +19,9 @@ Use targeted commands first:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
```
|
||||
|
||||
For WebUI checks:
|
||||
@@ -36,8 +36,8 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi
|
||||
Run the consolidated focused check when a change touches module discovery, optional integrations, shared mail components, or mailbox listing:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/check-focused.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/check-focused.sh
|
||||
```
|
||||
|
||||
## Working Rules
|
||||
|
||||
58
README.md
58
README.md
@@ -1,5 +1,13 @@
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions/workflows/module-matrix.yml)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions/workflows/dependency-audit.yml)
|
||||
# govoplan-core
|
||||
|
||||
<!-- govoplan-repository-type:start -->
|
||||
**Repository type:** system (kernel).
|
||||
<!-- govoplan-repository-type:end -->
|
||||
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=module-matrix.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=release-integration.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=dependency-audit.yml&actor=0&status=0)
|
||||
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=security-audit.yml&actor=0&status=0)
|
||||
|
||||
# govoplan-core
|
||||
|
||||
@@ -37,10 +45,12 @@ composition rules live in core only where they are stable kernel contracts.
|
||||
|
||||
## Backend development
|
||||
|
||||
Create or activate the core virtual environment, then install core and sibling modules from this repository:
|
||||
For whole-product development, create the virtualenv from the meta repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
python3 -m venv .venv
|
||||
./.venv/bin/python -m pip install --upgrade pip
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
@@ -48,7 +58,7 @@ Run the platform server from core through the module-aware development runner. T
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver \
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000
|
||||
```
|
||||
@@ -57,7 +67,7 @@ For example, to test campaign without files or mail:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \
|
||||
ENABLED_MODULES=access,campaigns /mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||
--host 127.0.0.1 \
|
||||
--port 8000
|
||||
```
|
||||
@@ -72,12 +82,29 @@ To run the production-like local profile with PostgreSQL, Redis, a Celery
|
||||
worker, explicit module configuration, and persistent local file storage:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
See [dev/production-like/README.md](dev/production-like/README.md) for ports,
|
||||
environment overrides, and cleanup commands.
|
||||
See `/mnt/DATA/git/govoplan/dev/production-like/README.md` for ports,
|
||||
environment overrides, and cleanup commands. Core keeps wrapper commands during
|
||||
the migration, but whole-product profiles are owned by the meta repository.
|
||||
|
||||
## Security audit
|
||||
|
||||
The repository includes a containerized audit toolbox for SAST, secret scanning,
|
||||
dependency checks, filesystem misconfiguration scans, duplication, and complexity
|
||||
reports. See [SECURITY_AUDIT.md](docs/SECURITY_AUDIT.md) for the operating model.
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||
```
|
||||
|
||||
CI runs the `ci` profile in report-only mode and uploads `audit-reports/` as an
|
||||
artifact. Once the baseline is clean, set `SECURITY_AUDIT_FAIL_ON_FINDINGS=1`
|
||||
or pass `--strict` locally to turn findings into a failing gate.
|
||||
|
||||
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
||||
|
||||
@@ -85,20 +112,23 @@ To verify the effective runtime paths and bootstrap behavior without starting uv
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
```
|
||||
|
||||
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
||||
|
||||
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||
The meta repository owns whole-product `requirements-dev.txt`,
|
||||
`requirements-release.txt`, and the root `.env.example` operator template. Core
|
||||
keeps package metadata and runtime commands. See
|
||||
[RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||
|
||||
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
||||
For self-hosted config bootstrap and validation:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||
```
|
||||
|
||||
## WebUI development
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
path_separator = os
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = sqlite:///./runtime/multimailer-dev.db
|
||||
sqlalchemy.url = sqlite:///./runtime/govoplan-dev.db
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""add audit outbox events to detailed dev track
|
||||
|
||||
Revision ID: 0f1e2d3c4b5a
|
||||
Revises: 4f2a9c8e7b6d
|
||||
Create Date: 2026-07-11 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0f1e2d3c4b5a"
|
||||
down_revision = "4f2a9c8e7b6d"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"audit_outbox_events",
|
||||
sa.Column("id", sa.String(length=36), nullable=False),
|
||||
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=200), nullable=False),
|
||||
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||
sa.Column("correlation_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("causation_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("classification", sa.String(length=40), nullable=False),
|
||||
sa.Column("payload", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=20), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("dispatched_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_outbox_events")),
|
||||
sa.UniqueConstraint("event_id", name="uq_audit_outbox_events_event_id"),
|
||||
)
|
||||
op.create_index("ix_audit_outbox_events_correlation_id", "audit_outbox_events", ["correlation_id"], unique=False)
|
||||
op.create_index("ix_audit_outbox_events_event_type", "audit_outbox_events", ["event_type"], unique=False)
|
||||
op.create_index(op.f("ix_audit_outbox_events_status"), "audit_outbox_events", ["status"], unique=False)
|
||||
op.create_index(
|
||||
"ix_audit_outbox_events_status_next_attempt_at",
|
||||
"audit_outbox_events",
|
||||
["status", "next_attempt_at"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("audit_outbox_events")
|
||||
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
@@ -0,0 +1,892 @@
|
||||
"""v0.1.7 core baseline
|
||||
|
||||
Revision ID: 4f2a9c8e7b6d
|
||||
Revises: None
|
||||
Create Date: 2026-07-11 00:00:00.000000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '4f2a9c8e7b6d'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _seed_core_defaults() -> None:
|
||||
bind = op.get_bind()
|
||||
now = _now()
|
||||
if not bind.execute(sa.text("SELECT 1 FROM core_system_settings WHERE id = 'global'")).first():
|
||||
settings_table = sa.table(
|
||||
"core_system_settings",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("default_locale", sa.String),
|
||||
sa.column("allow_tenant_custom_groups", sa.Boolean),
|
||||
sa.column("allow_tenant_custom_roles", sa.Boolean),
|
||||
sa.column("allow_tenant_api_keys", sa.Boolean),
|
||||
sa.column("settings", sa.JSON),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
bind.execute(
|
||||
settings_table.insert().values({
|
||||
"id": "global",
|
||||
"default_locale": "en",
|
||||
"allow_tenant_custom_groups": True,
|
||||
"allow_tenant_custom_roles": True,
|
||||
"allow_tenant_api_keys": True,
|
||||
"settings": {},
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}),
|
||||
)
|
||||
system_roles = {
|
||||
"system_owner": {
|
||||
"name": "System owner",
|
||||
"description": "Protected full instance-wide administration.",
|
||||
"permissions": ["system:*"],
|
||||
"is_builtin": True,
|
||||
},
|
||||
"system_admin": {
|
||||
"name": "System administrator",
|
||||
"description": "Manage the instance without granting the protected System owner role.",
|
||||
"permissions": [
|
||||
"system:tenants:read", "system:tenants:create", "system:tenants:update", "system:tenants:suspend",
|
||||
"system:accounts:read", "system:accounts:create", "system:accounts:update", "system:accounts:suspend",
|
||||
"system:roles:read", "system:roles:write", "system:roles:assign", "system:access:read", "system:access:assign",
|
||||
"system:audit:read", "system:settings:read", "system:settings:write", "system:governance:read", "system:governance:write",
|
||||
],
|
||||
"is_builtin": False,
|
||||
},
|
||||
"system_auditor": {
|
||||
"name": "System auditor",
|
||||
"description": "Read-only access to system administration and audit.",
|
||||
"permissions": [
|
||||
"system:tenants:read", "system:accounts:read", "system:roles:read", "system:access:read",
|
||||
"system:audit:read", "system:settings:read", "system:governance:read",
|
||||
],
|
||||
"is_builtin": False,
|
||||
},
|
||||
}
|
||||
roles_table = sa.table(
|
||||
"access_roles",
|
||||
sa.column("id", sa.String),
|
||||
sa.column("tenant_id", sa.String),
|
||||
sa.column("slug", sa.String),
|
||||
sa.column("name", sa.String),
|
||||
sa.column("description", sa.Text),
|
||||
sa.column("permissions", sa.JSON),
|
||||
sa.column("is_builtin", sa.Boolean),
|
||||
sa.column("is_assignable", sa.Boolean),
|
||||
sa.column("system_template_id", sa.String),
|
||||
sa.column("system_required", sa.Boolean),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
for slug, role in system_roles.items():
|
||||
if bind.execute(
|
||||
sa.text("SELECT 1 FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"),
|
||||
{"slug": slug},
|
||||
).first():
|
||||
continue
|
||||
bind.execute(
|
||||
roles_table.insert().values(
|
||||
id=str(uuid4()),
|
||||
tenant_id=None,
|
||||
slug=slug,
|
||||
name=role["name"],
|
||||
description=role["description"],
|
||||
permissions=role["permissions"],
|
||||
is_builtin=bool(role["is_builtin"]),
|
||||
is_assignable=True,
|
||||
system_template_id=None,
|
||||
system_required=False,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('core_scopes',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('allow_custom_groups', sa.Boolean(), nullable=True),
|
||||
sa.Column('allow_custom_roles', sa.Boolean(), nullable=True),
|
||||
sa.Column('allow_api_keys', sa.Boolean(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_scopes'))
|
||||
)
|
||||
op.create_index(op.f('ix_core_scopes_slug'), 'core_scopes', ['slug'], unique=True)
|
||||
op.create_table('access_accounts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('email', sa.String(length=320), nullable=False),
|
||||
sa.Column('normalized_email', sa.String(length=320), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||
sa.Column('password_reset_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_accounts'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_accounts_normalized_email'), 'access_accounts', ['normalized_email'], unique=True)
|
||||
op.create_table('access_groups',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_groups')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_access_groups_system_template_id'), 'access_groups', ['system_template_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False)
|
||||
op.create_table('access_roles',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||
sa.Column('is_builtin', sa.Boolean(), nullable=False),
|
||||
sa.Column('is_assignable', sa.Boolean(), nullable=False),
|
||||
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_roles')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_access_roles_system_template_id'), 'access_roles', ['system_template_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False)
|
||||
op.create_index('uq_roles_system_slug', 'access_roles', ['slug'], unique=True, sqlite_where=sa.text('tenant_id IS NULL'), postgresql_where=sa.text('tenant_id IS NULL'))
|
||||
op.create_table('admin_governance_templates',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('kind', sa.String(length=20), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_templates')),
|
||||
sa.UniqueConstraint('kind', 'slug', name='uq_governance_templates_kind_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_admin_governance_templates_kind'), 'admin_governance_templates', ['kind'], unique=False)
|
||||
op.create_table('attachment_blobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('mime_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('storage_bucket', sa.String(length=255), nullable=False),
|
||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
||||
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
||||
)
|
||||
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
||||
op.create_table('audit_outbox_events',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('event_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('event_type', sa.String(length=200), nullable=False),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('correlation_id', sa.String(length=128), nullable=True),
|
||||
sa.Column('causation_id', sa.String(length=128), nullable=True),
|
||||
sa.Column('classification', sa.String(length=40), nullable=False),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('status', sa.String(length=20), nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
||||
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('dispatched_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_outbox_events')),
|
||||
sa.UniqueConstraint('event_id', name='uq_audit_outbox_events_event_id')
|
||||
)
|
||||
op.create_index('ix_audit_outbox_events_correlation_id', 'audit_outbox_events', ['correlation_id'], unique=False)
|
||||
op.create_index('ix_audit_outbox_events_event_type', 'audit_outbox_events', ['event_type'], unique=False)
|
||||
op.create_index(op.f('ix_audit_outbox_events_status'), 'audit_outbox_events', ['status'], unique=False)
|
||||
op.create_index('ix_audit_outbox_events_status_next_attempt_at', 'audit_outbox_events', ['status', 'next_attempt_at'], unique=False)
|
||||
op.create_table('core_change_sequence',
|
||||
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||
sa.Column('resource_type', sa.String(length=100), nullable=False),
|
||||
sa.Column('resource_id', sa.String(length=255), nullable=False),
|
||||
sa.Column('operation', sa.String(length=30), nullable=False),
|
||||
sa.Column('actor_type', sa.String(length=30), nullable=True),
|
||||
sa.Column('actor_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence'))
|
||||
)
|
||||
op.create_index(op.f('ix_core_change_sequence_actor_id'), 'core_change_sequence', ['actor_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_actor_type'), 'core_change_sequence', ['actor_type'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_collection'), 'core_change_sequence', ['collection'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_collection_id', 'core_change_sequence', ['collection', 'id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_created_at'), 'core_change_sequence', ['created_at'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_module_id'), 'core_change_sequence', ['module_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_operation'), 'core_change_sequence', ['operation'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_resource', 'core_change_sequence', ['module_id', 'resource_type', 'resource_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_resource_id'), 'core_change_sequence', ['resource_id'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_resource_type'), 'core_change_sequence', ['resource_type'], unique=False)
|
||||
op.create_index(op.f('ix_core_change_sequence_tenant_id'), 'core_change_sequence', ['tenant_id'], unique=False)
|
||||
op.create_index('ix_core_change_sequence_tenant_module_id', 'core_change_sequence', ['tenant_id', 'module_id', 'id'], unique=False)
|
||||
op.create_table('core_change_sequence_retention_floor',
|
||||
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||
sa.Column('tenant_key', sa.String(length=36), nullable=False),
|
||||
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||
sa.Column('min_valid_sequence', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence_retention_floor')),
|
||||
sa.UniqueConstraint('tenant_key', 'module_id', 'collection', name='uq_core_change_sequence_retention_scope')
|
||||
)
|
||||
op.create_index('ix_core_change_sequence_retention_scope', 'core_change_sequence_retention_floor', ['tenant_key', 'module_id', 'collection'], unique=False)
|
||||
op.create_table('core_system_settings',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||
sa.Column('allow_tenant_custom_groups', sa.Boolean(), nullable=False),
|
||||
sa.Column('allow_tenant_custom_roles', sa.Boolean(), nullable=False),
|
||||
sa.Column('allow_tenant_api_keys', sa.Boolean(), nullable=False),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_system_settings'))
|
||||
)
|
||||
op.create_table('file_blobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('storage_backend', sa.String(length=50), nullable=False),
|
||||
sa.Column('storage_bucket', sa.String(length=255), nullable=True),
|
||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('ref_count', sa.Integer(), nullable=False),
|
||||
sa.Column('retained_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_blobs')),
|
||||
sa.UniqueConstraint('tenant_id', 'checksum_sha256', 'size_bytes', name='uq_file_blobs_tenant_checksum_size')
|
||||
)
|
||||
op.create_index(op.f('ix_file_blobs_checksum_sha256'), 'file_blobs', ['checksum_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_file_blobs_tenant_id'), 'file_blobs', ['tenant_id'], unique=False)
|
||||
op.create_table('access_group_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_group_role_assignments_group_id_access_groups'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_group_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_group_role_assignments')),
|
||||
sa.UniqueConstraint('tenant_id', 'group_id', 'role_id', name='uq_group_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_group_id'), 'access_group_role_assignments', ['group_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_role_id'), 'access_group_role_assignments', ['role_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_group_role_assignments_tenant_id'), 'access_group_role_assignments', ['tenant_id'], unique=False)
|
||||
op.create_table('access_system_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_system_role_assignments_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_system_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_system_role_assignments')),
|
||||
sa.UniqueConstraint('account_id', 'role_id', name='uq_system_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_system_role_assignments_account_id'), 'access_system_role_assignments', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_system_role_assignments_role_id'), 'access_system_role_assignments', ['role_id'], unique=False)
|
||||
op.create_table('access_users',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('email', sa.String(length=320), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
||||
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_users_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_users')),
|
||||
sa.UniqueConstraint('tenant_id', 'account_id', name='uq_users_tenant_account'),
|
||||
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
||||
)
|
||||
op.create_index(op.f('ix_access_users_account_id'), 'access_users', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False)
|
||||
op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False)
|
||||
op.create_table('admin_governance_template_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('template_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('mode', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['template_id'], ['admin_governance_templates.id'], name=op.f('fk_admin_governance_template_assignments_template_id_admin_governance_templates'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_template_assignments')),
|
||||
sa.UniqueConstraint('template_id', 'tenant_id', name='uq_governance_template_tenant')
|
||||
)
|
||||
op.create_index(op.f('ix_admin_governance_template_assignments_template_id'), 'admin_governance_template_assignments', ['template_id'], unique=False)
|
||||
op.create_index(op.f('ix_admin_governance_template_assignments_tenant_id'), 'admin_governance_template_assignments', ['tenant_id'], unique=False)
|
||||
op.create_table('access_api_keys',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('prefix', sa.String(length=16), nullable=False),
|
||||
sa.Column('key_hash', sa.String(length=128), nullable=False),
|
||||
sa.Column('scopes', sa.JSON(), nullable=False),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_api_keys_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_api_keys'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False)
|
||||
op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False)
|
||||
op.create_table('access_auth_sessions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('token_hash', sa.String(length=128), nullable=False),
|
||||
sa.Column('csrf_token_hash', sa.String(length=128), nullable=True),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('last_seen_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('user_agent', sa.String(length=500), nullable=True),
|
||||
sa.Column('ip_address', sa.String(length=100), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_auth_sessions_account_id_access_accounts'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_auth_sessions_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_auth_sessions'))
|
||||
)
|
||||
op.create_index(op.f('ix_access_auth_sessions_account_id'), 'access_auth_sessions', ['account_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_expires_at'), 'access_auth_sessions', ['expires_at'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_revoked_at'), 'access_auth_sessions', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_tenant_id'), 'access_auth_sessions', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_auth_sessions_token_hash'), 'access_auth_sessions', ['token_hash'], unique=True)
|
||||
op.create_index(op.f('ix_access_auth_sessions_user_id'), 'access_auth_sessions', ['user_id'], unique=False)
|
||||
op.create_table('access_user_group_memberships',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_user_group_memberships_group_id_access_groups'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_group_memberships_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_group_memberships')),
|
||||
sa.UniqueConstraint('tenant_id', 'user_id', 'group_id', name='uq_user_group_memberships')
|
||||
)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_group_id'), 'access_user_group_memberships', ['group_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_tenant_id'), 'access_user_group_memberships', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_group_memberships_user_id'), 'access_user_group_memberships', ['user_id'], unique=False)
|
||||
op.create_table('access_user_role_assignments',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_user_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_role_assignments_user_id_access_users'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_role_assignments')),
|
||||
sa.UniqueConstraint('tenant_id', 'user_id', 'role_id', name='uq_user_role_assignments')
|
||||
)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_role_id'), 'access_user_role_assignments', ['role_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_tenant_id'), 'access_user_role_assignments', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_access_user_role_assignments_user_id'), 'access_user_role_assignments', ['user_id'], unique=False)
|
||||
op.create_table('campaigns',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('external_id', sa.String(length=255), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('settings', sa.JSON(), nullable=False),
|
||||
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_campaigns_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_campaigns_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
||||
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
||||
)
|
||||
op.create_index(op.f('ix_campaigns_created_by_user_id'), 'campaigns', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_external_id'), 'campaigns', ['external_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_owner_group_id'), 'campaigns', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_owner_user_id'), 'campaigns', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_status'), 'campaigns', ['status'], unique=False)
|
||||
op.create_index(op.f('ix_campaigns_tenant_id'), 'campaigns', ['tenant_id'], unique=False)
|
||||
op.create_table('file_assets',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('display_path', sa.String(length=1000), nullable=False),
|
||||
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_assets_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_assets_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_assets_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_assets'))
|
||||
)
|
||||
op.create_index(op.f('ix_file_assets_created_by_user_id'), 'file_assets', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_current_version_id'), 'file_assets', ['current_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_deleted_at'), 'file_assets', ['deleted_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_display_path'), 'file_assets', ['display_path'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_filename'), 'file_assets', ['filename'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_group_id'), 'file_assets', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_type'), 'file_assets', ['owner_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_owner_user_id'), 'file_assets', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_assets_tenant_id'), 'file_assets', ['tenant_id'], unique=False)
|
||||
op.create_table('file_folders',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('path', sa.String(length=1000), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_folders_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_folders_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_folders_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_folders'))
|
||||
)
|
||||
op.create_index(op.f('ix_file_folders_created_by_user_id'), 'file_folders', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_deleted_at'), 'file_folders', ['deleted_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_group_id'), 'file_folders', ['owner_group_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_type'), 'file_folders', ['owner_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_owner_user_id'), 'file_folders', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_path'), 'file_folders', ['path'], unique=False)
|
||||
op.create_index(op.f('ix_file_folders_tenant_id'), 'file_folders', ['tenant_id'], unique=False)
|
||||
op.create_index('uq_file_folders_active_group_path', 'file_folders', ['tenant_id', 'owner_group_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"))
|
||||
op.create_index('uq_file_folders_active_user_path', 'file_folders', ['tenant_id', 'owner_user_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"))
|
||||
op.create_table('mail_server_profiles',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('scope_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('scope_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||
sa.Column('smtp_config', sa.JSON(), nullable=False),
|
||||
sa.Column('smtp_username', sa.String(length=320), nullable=True),
|
||||
sa.Column('smtp_password_encrypted', sa.Text(), nullable=True),
|
||||
sa.Column('imap_config', sa.JSON(), nullable=True),
|
||||
sa.Column('imap_username', sa.String(length=320), nullable=True),
|
||||
sa.Column('imap_password_encrypted', sa.Text(), nullable=True),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('updated_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['updated_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_updated_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_mail_server_profiles')),
|
||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_mail_server_profiles_tenant_slug')
|
||||
)
|
||||
op.create_index(op.f('ix_mail_server_profiles_created_by_user_id'), 'mail_server_profiles', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_is_active'), 'mail_server_profiles', ['is_active'], unique=False)
|
||||
op.create_index('ix_mail_server_profiles_scope', 'mail_server_profiles', ['scope_type', 'scope_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_scope_id'), 'mail_server_profiles', ['scope_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_scope_type'), 'mail_server_profiles', ['scope_type'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_tenant_id'), 'mail_server_profiles', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_mail_server_profiles_updated_by_user_id'), 'mail_server_profiles', ['updated_by_user_id'], unique=False)
|
||||
op.create_table('attachment_instances',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('logical_name', sa.String(length=500), nullable=True),
|
||||
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||
sa.Column('tags', sa.JSON(), nullable=False),
|
||||
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
||||
)
|
||||
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_campaign_id'), 'attachment_instances', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_owner_user_id'), 'attachment_instances', ['owner_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_attachment_instances_tenant_id'), 'attachment_instances', ['tenant_id'], unique=False)
|
||||
op.create_table('audit_log',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('scope', sa.String(length=20), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('api_key_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('action', sa.String(length=100), nullable=False),
|
||||
sa.Column('object_type', sa.String(length=100), nullable=True),
|
||||
sa.Column('object_id', sa.String(length=100), nullable=True),
|
||||
sa.Column('details', sa.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_access_api_keys'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
||||
)
|
||||
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_api_key_id'), 'audit_log', ['api_key_id'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_object_id'), 'audit_log', ['object_id'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_object_type'), 'audit_log', ['object_type'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_scope'), 'audit_log', ['scope'], unique=False)
|
||||
op.create_index('ix_audit_log_scope_created_at', 'audit_log', ['scope', 'created_at'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_tenant_id'), 'audit_log', ['tenant_id'], unique=False)
|
||||
op.create_index('ix_audit_log_tenant_scope_created_at', 'audit_log', ['tenant_id', 'scope', 'created_at'], unique=False)
|
||||
op.create_index(op.f('ix_audit_log_user_id'), 'audit_log', ['user_id'], unique=False)
|
||||
op.create_table('campaign_shares',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_shares_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_shares')),
|
||||
sa.UniqueConstraint('campaign_id', 'target_type', 'target_id', name='uq_campaign_share_target')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_shares_campaign_id'), 'campaign_shares', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_created_by_user_id'), 'campaign_shares', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_revoked_at'), 'campaign_shares', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_target_id'), 'campaign_shares', ['target_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_target_type'), 'campaign_shares', ['target_type'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_shares_tenant_id'), 'campaign_shares', ['tenant_id'], unique=False)
|
||||
op.create_table('campaign_versions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||
sa.Column('raw_json', sa.JSON(), nullable=False),
|
||||
sa.Column('schema_version', sa.String(length=50), nullable=False),
|
||||
sa.Column('source_filename', sa.String(length=500), nullable=True),
|
||||
sa.Column('source_base_path', sa.String(length=1000), nullable=True),
|
||||
sa.Column('workflow_state', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_flow', sa.String(length=50), nullable=False),
|
||||
sa.Column('current_step', sa.String(length=100), nullable=True),
|
||||
sa.Column('is_complete', sa.Boolean(), nullable=False),
|
||||
sa.Column('editor_state', sa.JSON(), nullable=False),
|
||||
sa.Column('autosaved_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('locked_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('user_lock_state', sa.String(length=20), nullable=True),
|
||||
sa.Column('user_locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('user_locked_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('validation_summary', sa.JSON(), nullable=True),
|
||||
sa.Column('build_summary', sa.JSON(), nullable=True),
|
||||
sa.Column('execution_snapshot', sa.JSON(), nullable=True),
|
||||
sa.Column('execution_snapshot_hash', sa.String(length=64), nullable=True),
|
||||
sa.Column('execution_snapshot_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_versions_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['user_locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_user_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_versions')),
|
||||
sa.UniqueConstraint('campaign_id', 'version_number', name='uq_campaign_versions_campaign_number')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_versions_campaign_id'), 'campaign_versions', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_current_flow'), 'campaign_versions', ['current_flow'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_execution_snapshot_hash'), 'campaign_versions', ['execution_snapshot_hash'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_locked_by_user_id'), 'campaign_versions', ['locked_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_user_lock_state'), 'campaign_versions', ['user_lock_state'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_user_locked_by_user_id'), 'campaign_versions', ['user_locked_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_versions_workflow_state'), 'campaign_versions', ['workflow_state'], unique=False)
|
||||
op.create_table('file_shares',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_shares_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_shares')),
|
||||
sa.UniqueConstraint('file_asset_id', 'target_type', 'target_id', 'revoked_at', name='uq_file_shares_active_target')
|
||||
)
|
||||
op.create_index(op.f('ix_file_shares_created_by_user_id'), 'file_shares', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_file_asset_id'), 'file_shares', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_revoked_at'), 'file_shares', ['revoked_at'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_target_id'), 'file_shares', ['target_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_target_type'), 'file_shares', ['target_type'], unique=False)
|
||||
op.create_index(op.f('ix_file_shares_tenant_id'), 'file_shares', ['tenant_id'], unique=False)
|
||||
op.create_table('file_versions',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||
sa.Column('filename_at_upload', sa.String(length=500), nullable=False),
|
||||
sa.Column('display_path_at_upload', sa.String(length=1000), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['file_blobs.id'], name=op.f('fk_file_versions_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_versions_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_versions_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_versions')),
|
||||
sa.UniqueConstraint('file_asset_id', 'version_number', name='uq_file_versions_asset_number')
|
||||
)
|
||||
op.create_index(op.f('ix_file_versions_blob_id'), 'file_versions', ['blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_checksum_sha256'), 'file_versions', ['checksum_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_created_by_user_id'), 'file_versions', ['created_by_user_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_file_asset_id'), 'file_versions', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_file_versions_tenant_id'), 'file_versions', ['tenant_id'], unique=False)
|
||||
op.create_table('campaign_jobs',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('entry_index', sa.Integer(), nullable=False),
|
||||
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('recipient_email', sa.String(length=320), nullable=True),
|
||||
sa.Column('subject', sa.String(length=998), nullable=True),
|
||||
sa.Column('message_id_header', sa.String(length=255), nullable=True),
|
||||
sa.Column('eml_storage_key', sa.String(length=1000), nullable=True),
|
||||
sa.Column('eml_local_path', sa.String(length=1000), nullable=True),
|
||||
sa.Column('eml_size_bytes', sa.Integer(), nullable=True),
|
||||
sa.Column('eml_sha256', sa.String(length=64), nullable=True),
|
||||
sa.Column('build_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('validation_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('queue_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('send_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('imap_status', sa.String(length=50), nullable=False),
|
||||
sa.Column('attempt_count', sa.Integer(), nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('queued_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claimed_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||
sa.Column('smtp_started_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('outcome_unknown_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('sent_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('resolved_recipients', sa.JSON(), nullable=True),
|
||||
sa.Column('resolved_attachments', sa.JSON(), nullable=False),
|
||||
sa.Column('issues_snapshot', sa.JSON(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
||||
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_jobs_build_status'), 'campaign_jobs', ['build_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_campaign_id'), 'campaign_jobs', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_campaign_version_id'), 'campaign_jobs', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_claim_token'), 'campaign_jobs', ['claim_token'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_eml_sha256'), 'campaign_jobs', ['eml_sha256'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_entry_id'), 'campaign_jobs', ['entry_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_imap_status'), 'campaign_jobs', ['imap_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_queue_status'), 'campaign_jobs', ['queue_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_recipient_email'), 'campaign_jobs', ['recipient_email'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_send_status'), 'campaign_jobs', ['send_status'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_tenant_id'), 'campaign_jobs', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_jobs_validation_status'), 'campaign_jobs', ['validation_status'], unique=False)
|
||||
op.create_table('campaign_attachment_uses',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_job_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('entry_index', sa.Integer(), nullable=True),
|
||||
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_version_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('file_blob_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('filename_used', sa.String(length=500), nullable=False),
|
||||
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('use_stage', sa.String(length=20), nullable=False),
|
||||
sa.Column('used_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_attachment_uses_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_attachment_uses_campaign_job_id_campaign_jobs'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_attachment_uses_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_campaign_attachment_uses_file_asset_id_file_assets'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['file_blob_id'], ['file_blobs.id'], name=op.f('fk_campaign_attachment_uses_file_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||
sa.ForeignKeyConstraint(['file_version_id'], ['file_versions.id'], name=op.f('fk_campaign_attachment_uses_file_version_id_file_versions'), ondelete='RESTRICT'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_attachment_uses')),
|
||||
sa.UniqueConstraint('campaign_job_id', 'file_version_id', 'filename_used', 'use_stage', name='uq_campaign_attachment_uses_job_file_stage')
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_id'), 'campaign_attachment_uses', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_job_id'), 'campaign_attachment_uses', ['campaign_job_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_campaign_version_id'), 'campaign_attachment_uses', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_entry_id'), 'campaign_attachment_uses', ['entry_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_asset_id'), 'campaign_attachment_uses', ['file_asset_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_blob_id'), 'campaign_attachment_uses', ['file_blob_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_file_version_id'), 'campaign_attachment_uses', ['file_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_tenant_id'), 'campaign_attachment_uses', ['tenant_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_use_stage'), 'campaign_attachment_uses', ['use_stage'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_attachment_uses_used_at'), 'campaign_attachment_uses', ['used_at'], unique=False)
|
||||
op.create_table('campaign_issues',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('campaign_version_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=True),
|
||||
sa.Column('severity', sa.String(length=20), nullable=False),
|
||||
sa.Column('code', sa.String(length=100), nullable=False),
|
||||
sa.Column('message', sa.Text(), nullable=False),
|
||||
sa.Column('source', sa.String(length=255), nullable=True),
|
||||
sa.Column('behavior', sa.String(length=50), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
||||
)
|
||||
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_campaign_version_id'), 'campaign_issues', ['campaign_version_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_code'), 'campaign_issues', ['code'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_job_id'), 'campaign_issues', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_severity'), 'campaign_issues', ['severity'], unique=False)
|
||||
op.create_index(op.f('ix_campaign_issues_tenant_id'), 'campaign_issues', ['tenant_id'], unique=False)
|
||||
op.create_table('imap_append_attempts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||
sa.Column('folder', sa.String(length=500), nullable=True),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_imap_append_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_imap_append_attempts'))
|
||||
)
|
||||
op.create_index(op.f('ix_imap_append_attempts_job_id'), 'imap_append_attempts', ['job_id'], unique=False)
|
||||
op.create_table('send_attempts',
|
||||
sa.Column('id', sa.String(length=36), nullable=False),
|
||||
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(length=50), nullable=False),
|
||||
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||
sa.Column('smtp_status_code', sa.Integer(), nullable=True),
|
||||
sa.Column('smtp_response', sa.Text(), nullable=True),
|
||||
sa.Column('error_type', sa.String(length=255), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_send_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_send_attempts'))
|
||||
)
|
||||
op.create_index(op.f('ix_send_attempts_claim_token'), 'send_attempts', ['claim_token'], unique=False)
|
||||
op.create_index(op.f('ix_send_attempts_job_id'), 'send_attempts', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_send_attempts_status'), 'send_attempts', ['status'], unique=False)
|
||||
_seed_core_defaults()
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table('send_attempts')
|
||||
op.drop_table('imap_append_attempts')
|
||||
op.drop_table('campaign_issues')
|
||||
op.drop_table('campaign_attachment_uses')
|
||||
op.drop_table('campaign_jobs')
|
||||
op.drop_table('file_versions')
|
||||
op.drop_table('file_shares')
|
||||
op.drop_table('campaign_versions')
|
||||
op.drop_table('campaign_shares')
|
||||
op.drop_table('audit_log')
|
||||
op.drop_table('attachment_instances')
|
||||
op.drop_table('mail_server_profiles')
|
||||
op.drop_table('file_folders')
|
||||
op.drop_table('file_assets')
|
||||
op.drop_table('campaigns')
|
||||
op.drop_table('access_user_role_assignments')
|
||||
op.drop_table('access_user_group_memberships')
|
||||
op.drop_table('access_auth_sessions')
|
||||
op.drop_table('access_api_keys')
|
||||
op.drop_table('admin_governance_template_assignments')
|
||||
op.drop_table('access_users')
|
||||
op.drop_table('access_system_role_assignments')
|
||||
op.drop_table('access_group_role_assignments')
|
||||
op.drop_table('file_blobs')
|
||||
op.drop_table('core_system_settings')
|
||||
op.drop_table('core_change_sequence_retention_floor')
|
||||
op.drop_table('core_change_sequence')
|
||||
op.drop_table('audit_outbox_events')
|
||||
op.drop_table('attachment_blobs')
|
||||
op.drop_table('admin_governance_templates')
|
||||
op.drop_table('access_roles')
|
||||
op.drop_table('access_groups')
|
||||
op.drop_table('access_accounts')
|
||||
op.drop_table('core_scopes')
|
||||
@@ -1,5 +0,0 @@
|
||||
GOVOPLAN_POSTGRES_DB=govoplan
|
||||
GOVOPLAN_POSTGRES_USER=govoplan
|
||||
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
|
||||
GOVOPLAN_POSTGRES_PORT=55432
|
||||
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan
|
||||
@@ -1,126 +0,0 @@
|
||||
# PostgreSQL Development Profile
|
||||
|
||||
GovOPlaN development now defaults to PostgreSQL:
|
||||
|
||||
```text
|
||||
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||
```
|
||||
|
||||
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
|
||||
|
||||
```text
|
||||
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||
```
|
||||
|
||||
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
|
||||
The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if
|
||||
you need a different database.
|
||||
|
||||
## Host PostgreSQL Setup
|
||||
|
||||
Create the default local role and database from a host shell:
|
||||
|
||||
```bash
|
||||
export GOVOPLAN_PG_DB=govoplan_dev
|
||||
export GOVOPLAN_PG_USER=govoplan_dev
|
||||
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
|
||||
echo
|
||||
|
||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
|
||||
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
|
||||
else
|
||||
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
|
||||
fi
|
||||
|
||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
|
||||
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
|
||||
fi
|
||||
|
||||
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
|
||||
|
||||
touch ~/.pgpass
|
||||
chmod 600 ~/.pgpass
|
||||
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
|
||||
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
|
||||
mv ~/.pgpass.tmp ~/.pgpass
|
||||
chmod 600 ~/.pgpass
|
||||
```
|
||||
|
||||
Check access:
|
||||
|
||||
```bash
|
||||
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
|
||||
-c 'select current_user, current_database();'
|
||||
```
|
||||
|
||||
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
|
||||
available through:
|
||||
|
||||
```bash
|
||||
flatpak-spawn --host /usr/bin/psql --version
|
||||
flatpak-spawn --host /usr/bin/pg_dump --version
|
||||
flatpak-spawn --host /usr/bin/pg_restore --version
|
||||
```
|
||||
|
||||
## Run GovOPlaN Against Host PostgreSQL
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||
--with-dev-data
|
||||
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
```
|
||||
|
||||
For the full backend and WebUI launcher:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-dev.sh
|
||||
```
|
||||
|
||||
To force the old SQLite fallback for a disposable local run:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh
|
||||
```
|
||||
|
||||
## Disposable Docker Testbed
|
||||
|
||||
This testbed is for migration and module-permutation checks. It is not a
|
||||
production deployment profile.
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
```
|
||||
|
||||
Run the integration check from the core checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
|
||||
`--reset-schema` drops and recreates the `public` schema before every module
|
||||
set. Use it only against this disposable database.
|
||||
|
||||
Stop the testbed:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Remove all test data:
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
@@ -1,22 +0,0 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: govoplan-core-postgres-dev
|
||||
environment:
|
||||
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
|
||||
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
|
||||
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
@@ -1,27 +0,0 @@
|
||||
APP_ENV=staging
|
||||
GOVOPLAN_INSTALL_PROFILE=production-like
|
||||
MASTER_KEY_B64=
|
||||
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
|
||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
|
||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
|
||||
|
||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
|
||||
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
REDIS_URL=redis://127.0.0.1:56379/0
|
||||
CELERY_ENABLED=true
|
||||
CELERY_QUEUES=send_email,append_sent,default
|
||||
|
||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
||||
AUTH_COOKIE_SECURE=false
|
||||
FILE_STORAGE_BACKEND=local
|
||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
||||
DEV_AUTO_MIGRATE_ENABLED=false
|
||||
DEV_BOOTSTRAP_ENABLED=true
|
||||
@@ -1,60 +0,0 @@
|
||||
# Production-Like Development Profile
|
||||
|
||||
This profile runs the shared services that production depends on while keeping
|
||||
API, worker, and WebUI code in the editable local repositories.
|
||||
|
||||
It provides:
|
||||
|
||||
- PostgreSQL with a persistent Docker volume
|
||||
- Redis with append-only persistence
|
||||
- explicit `ENABLED_MODULES`
|
||||
- local durable file storage under `runtime/production-like/files`
|
||||
- a Celery worker process using the same queues as the API
|
||||
|
||||
Start it from the core repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
The helper wrapper exposes repeatable lifecycle commands:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
scripts/production-like-dev.sh stop
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The launcher uses `dev/production-like/.env` when present, otherwise
|
||||
`dev/production-like/.env.example`. Copy the example when you want local port or
|
||||
password changes:
|
||||
|
||||
```bash
|
||||
cp dev/production-like/.env.example dev/production-like/.env
|
||||
```
|
||||
|
||||
The API and worker use:
|
||||
|
||||
```text
|
||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||
REDIS_URL=redis://127.0.0.1:56379/0
|
||||
CELERY_ENABLED=true
|
||||
```
|
||||
|
||||
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
|
||||
containers keep running by default so the next launch is fast. To stop them too:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
To remove all profile data:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
```
|
||||
@@ -1,37 +0,0 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: govoplan-production-like-postgres
|
||||
environment:
|
||||
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
|
||||
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
|
||||
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: govoplan-production-like-redis
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
ports:
|
||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
redis-data:
|
||||
@@ -1,6 +1,6 @@
|
||||
# GovOPlaN RBAC And Resource-Access Model
|
||||
|
||||
**Updated:** 2026-07-09
|
||||
**Updated:** 2026-07-11
|
||||
|
||||
## Authorization Equation
|
||||
|
||||
@@ -254,6 +254,61 @@ space/folder/file ownership:
|
||||
External file connections and spaces are additionally constrained by connector
|
||||
policy and owner/group assignment in the files module.
|
||||
|
||||
## Resource Access Explanations
|
||||
|
||||
The access module exposes a diagnostic endpoint for explaining why a principal
|
||||
can see or operate on a concrete resource:
|
||||
|
||||
```text
|
||||
GET /api/v1/admin/access/resource-explanation
|
||||
```
|
||||
|
||||
Required query values:
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `user_id` | Tenant membership to explain. The current module UIs pass the signed-in user. |
|
||||
| `resource_type` | Module-owned type such as `file`, `folder`, or `campaign`. |
|
||||
| `resource_id` | Stable module resource identifier. |
|
||||
| `action` | Permission/action being explained, for example `files:file:read`. |
|
||||
| `tenant_id` | Optional tenant override for system/admin contexts. |
|
||||
|
||||
The access module always contributes effective-scope provenance for the action.
|
||||
Installed modules may add resource provenance by exposing a
|
||||
`ResourceAccessExplanationProvider` through a capability consumed by access.
|
||||
Providers should return only facts they own, using these provenance kinds:
|
||||
|
||||
| Kind | Meaning |
|
||||
| --- | --- |
|
||||
| `resource` | The concrete resource or an explicit not-found result. |
|
||||
| `owner` | Matching user/group ownership. |
|
||||
| `share` | Matching explicit user/group/tenant share. |
|
||||
| `policy` | Administrative bypass or policy-derived grant. |
|
||||
| `role` / `right` | Scope and role provenance from access itself. |
|
||||
|
||||
Files currently registers `files.access` and explains file assets plus folders.
|
||||
Persisted folders use their database ID. Folder rows inferred from file paths use
|
||||
a deterministic virtual ID:
|
||||
|
||||
```text
|
||||
virtual-folder:v1:<tenant_id>:<owner_type>:<owner_id>:<base64url(normalized_path)>
|
||||
```
|
||||
|
||||
The files provider validates virtual folder IDs before returning provenance: the
|
||||
tenant and owner must match the resource ID, and at least one active file must
|
||||
exist below the normalized folder path. This keeps virtual folder explanations
|
||||
stable without forcing every inferred tree node to become a stored folder row.
|
||||
|
||||
Campaign currently registers `campaigns.access` and explains the campaign
|
||||
ownership/sharing object itself. Finer-grained campaign sub-objects are tracked
|
||||
separately in `govoplan-campaign#50` until their resource identifiers and access
|
||||
rules are decided.
|
||||
|
||||
Cross-user resource explanation is a policy feature, not a module-local UI
|
||||
detail. Until `govoplan-policy#6` is resolved, module UIs should default to
|
||||
current-user explanation and avoid importing access-admin user-picking
|
||||
components.
|
||||
|
||||
## Mail Servers
|
||||
|
||||
| Scope | Meaning |
|
||||
|
||||
@@ -58,18 +58,18 @@ Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.
|
||||
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/check-focused.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/check-focused.sh
|
||||
```
|
||||
|
||||
For smaller changes, prefer the narrow command named in the relevant `AGENTS.md` file. Examples:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||
|
||||
cd /mnt/DATA/git/govoplan-mail
|
||||
/mnt/DATA/git/govoplan-core/.venv/bin/python -m unittest discover -s tests
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest discover -s tests
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core/webui
|
||||
PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations
|
||||
|
||||
@@ -6,23 +6,24 @@ metadata and can fail for newly disclosed advisories without a source change.
|
||||
|
||||
## Local Workflow
|
||||
|
||||
Install the development audit dependency once:
|
||||
Install the whole-product development dependencies once from the meta
|
||||
repository:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Run both backend and WebUI production audits:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
The script runs:
|
||||
|
||||
- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||
- `tools/checks/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||
legacy editable package metadata, deprecated framework constants, and the
|
||||
Starlette `TestClient` deprecation smoke when test dependencies are present
|
||||
- `python -m pip_audit --progress-spinner off`
|
||||
@@ -31,11 +32,12 @@ The script runs:
|
||||
For fast local checks without vulnerability metadata lookups, run:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 \
|
||||
bash tools/checks/check-dependency-hygiene.sh
|
||||
```
|
||||
|
||||
This is also part of `scripts/check-focused.sh`, so resolver drift and
|
||||
This is also part of `govoplan/tools/checks/check-focused.sh`, so resolver drift and
|
||||
deprecation regressions fail close to the code change that introduced them.
|
||||
|
||||
Override tool paths when testing from a disposable environment:
|
||||
@@ -43,12 +45,13 @@ Override tool paths when testing from a disposable environment:
|
||||
```bash
|
||||
PYTHON=/tmp/govoplan-audit/bin/python \
|
||||
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
||||
bash scripts/check-dependency-audits.sh
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core \
|
||||
bash /mnt/DATA/git/govoplan/tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
## CI Workflow
|
||||
|
||||
`.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||
`govoplan/.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
||||
pull requests, and a weekly schedule.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ installer CLI/daemon for package mutation under maintenance mode.
|
||||
Generate a deployment-local template:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||
--profile self-hosted \
|
||||
--generate-secrets \
|
||||
@@ -54,8 +54,9 @@ PY
|
||||
| Setting | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
||||
| `GOVOPLAN_MIGRATION_TRACK` | `release` | Use the release track for normal runtime and deployments. Use `dev` only for fresh/disposable databases that intentionally replay detailed development migrations. |
|
||||
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
||||
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `govoplan/tools/launch/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||
|
||||
Operator rule: take a database backup before applying migrations or destructive
|
||||
module retirement. For non-SQLite databases, configure deployment-specific
|
||||
@@ -68,10 +69,11 @@ supported only for tiny disposable profiles and unit-test style smoke runs.
|
||||
Production/staging deployments should use a managed PostgreSQL database and
|
||||
explicit migration commands.
|
||||
|
||||
Install the server extra so the `psycopg` driver is available:
|
||||
Install the full release profile from the meta repository so core, modules, and
|
||||
the `psycopg` driver are available:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
@@ -113,30 +115,30 @@ pg_restore --clean --if-exists \
|
||||
```
|
||||
|
||||
For local development, create the host database described in
|
||||
`dev/postgres/README.md`, then run:
|
||||
`/mnt/DATA/git/govoplan/dev/postgres/README.md`, then run:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||
--with-dev-data
|
||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||
scripts/launch-dev.sh
|
||||
tools/launch/launch-dev.sh
|
||||
```
|
||||
|
||||
For disposable local validation against a throwaway PostgreSQL instance, use
|
||||
the bundled PostgreSQL testbed:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
tools/checks/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
@@ -218,7 +220,7 @@ configuration, not the core runtime contract. Store them in a local ignored
|
||||
## First Deployment Flow
|
||||
|
||||
1. Create an environment file or secret set with the runtime contract above.
|
||||
2. Install the tagged core and module packages from `requirements-release.txt`.
|
||||
2. Install the tagged core and module packages from meta `requirements-release.txt`.
|
||||
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
||||
artifact from the same release tag.
|
||||
4. Run database migrations with the target `DATABASE_URL`.
|
||||
@@ -251,22 +253,22 @@ WebUI code in the editable repositories while Docker provides PostgreSQL and
|
||||
Redis:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/launch-production-like-dev.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
The helper wrapper provides explicit lifecycle commands:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
scripts/production-like-dev.sh stop
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
tools/launch/production-like-dev.sh validate-config
|
||||
tools/launch/production-like-dev.sh seed
|
||||
tools/launch/production-like-dev.sh start
|
||||
tools/launch/production-like-dev.sh stop
|
||||
tools/launch/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The launcher uses `dev/production-like/.env` when present, otherwise the checked
|
||||
in `.env.example`. It runs:
|
||||
The launcher uses `govoplan/dev/production-like/.env` when present, otherwise
|
||||
the checked in `.env.example`. It runs:
|
||||
|
||||
- PostgreSQL on `127.0.0.1:55433`
|
||||
- Redis on `127.0.0.1:56379`
|
||||
@@ -286,7 +288,7 @@ deployment test.
|
||||
To stop PostgreSQL and Redis when the launcher exits:
|
||||
|
||||
```bash
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 tools/launch/launch-production-like-dev.sh
|
||||
```
|
||||
|
||||
## Module Install/Uninstall Operations
|
||||
@@ -392,7 +394,7 @@ Run the rollback drill before relying on installer automation in a new
|
||||
environment:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/module-installer-rollback-drill.py --format json
|
||||
/mnt/DATA/git/govoplan/tools/checks/module-installer-rollback-drill.py --format json
|
||||
```
|
||||
|
||||
The drill uses temporary SQLite databases and simulated package commands. It
|
||||
|
||||
@@ -133,13 +133,12 @@ Access:
|
||||
|
||||
Tenancy:
|
||||
|
||||
- `tenancy.tenant.created`
|
||||
- `tenancy.tenant.updated`
|
||||
- `tenancy.tenant.suspended`
|
||||
- `tenancy.tenant.reactivated`
|
||||
- `tenancy.tenant.delete_requested`
|
||||
- `tenancy.tenant.delete_blocked`
|
||||
- `tenancy.tenant.deleted`
|
||||
- `tenant.created`
|
||||
- `tenant.updated`
|
||||
- `tenant.suspended`
|
||||
- `tenant.resumed`
|
||||
- `tenant.deletion_requested`
|
||||
- `tenant.erasure_completed`
|
||||
|
||||
Policy:
|
||||
|
||||
|
||||
@@ -1,319 +1,16 @@
|
||||
# Gitea Issues And Wiki Workflow
|
||||
|
||||
Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes.
|
||||
The shared GovOPlaN Gitea issue, label, and wiki workflow tooling moved to the
|
||||
meta repository.
|
||||
|
||||
The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state.
|
||||
|
||||
## Initial Setup
|
||||
|
||||
The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`.
|
||||
|
||||
The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan-core.git`). Override inference when needed:
|
||||
Use:
|
||||
|
||||
```bash
|
||||
export GITEA_URL=https://git.add-ideas.de
|
||||
export GITEA_OWNER=add-ideas
|
||||
export GITEA_REPO=govoplan-core
|
||||
export GITEA_TOKEN=...
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/gitea/gitea-sync-labels.py --help
|
||||
tools/gitea/gitea-sync-wiki.py --help
|
||||
```
|
||||
|
||||
The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens:
|
||||
Canonical documentation:
|
||||
|
||||
```bash
|
||||
GITEA_TOKEN=...
|
||||
# Optional if origin inference is not enough:
|
||||
GITEA_URL=https://git.add-ideas.de
|
||||
GITEA_OWNER=add-ideas
|
||||
GITEA_REPO=govoplan-core
|
||||
```
|
||||
|
||||
For a shared credentials file outside the target repository, pass `--env-file`:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
||||
```
|
||||
|
||||
Create a Gitea token with issue read/write access and label-management
|
||||
permission for the repository. On scoped-token instances, this usually means
|
||||
issue read/write and, if label writes are rejected, repository write permission
|
||||
too.
|
||||
|
||||
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
|
||||
Creating or updating organization labels requires a token with
|
||||
`write:organization`. Repository label management only needs repository label
|
||||
permission, but it duplicates the taxonomy into each repository.
|
||||
|
||||
Preview and apply labels:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py
|
||||
./scripts/gitea-sync-labels.py --apply
|
||||
```
|
||||
|
||||
Preview and apply the shared taxonomy as organization labels:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
|
||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
The import helpers resolve repository labels and organization labels. Repository
|
||||
labels win when a repository defines the same name locally, but a repository does
|
||||
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
|
||||
`module/*`, `area/*`, `source/*`, or `codex/*` label.
|
||||
|
||||
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
||||
|
||||
## Multiple Repositories And Workspaces
|
||||
|
||||
The helper scripts are path-based and can run from this core checkout against any repository with a Gitea remote:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply
|
||||
./scripts/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail
|
||||
./scripts/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress
|
||||
```
|
||||
|
||||
Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH.
|
||||
|
||||
When one core checkout drives another workspace, prefer `--env-file` for shared credentials instead of putting `GITEA_REPO` in the core `.env`; a repo-specific `GITEA_REPO` can accidentally override target inference.
|
||||
|
||||
For non-GovOPlaN projects, provide a project-specific label file and module/project label:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-labels.py \
|
||||
--root /path/to/project \
|
||||
--labels-file /path/to/project/docs/gitea-labels.json \
|
||||
--apply
|
||||
|
||||
./scripts/gitea-todo-import.py \
|
||||
--root /path/to/project \
|
||||
--module-label project/example \
|
||||
--extra-label area/backend
|
||||
```
|
||||
|
||||
If another project does not use `area/*` labels, disable area inference:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-todo-import.py \
|
||||
--root /path/to/project \
|
||||
--module-label project/example \
|
||||
--no-area-labels
|
||||
```
|
||||
|
||||
Install or refresh the shared issue templates in sibling or external repositories:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail
|
||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply
|
||||
```
|
||||
|
||||
The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-<name>` checkout maps to `module/<name>`. For another workspace or repository name, pass an explicit label:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-install-workflow.py /path/to/repo --module-label module/example --apply
|
||||
```
|
||||
|
||||
Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in core and run the sync script from core.
|
||||
|
||||
For a fully portable workflow kit, copy these files into the other project:
|
||||
|
||||
- `scripts/gitea_common.py`
|
||||
- `scripts/gitea-sync-labels.py`
|
||||
- `scripts/gitea-todo-import.py`
|
||||
- `scripts/gitea-codex-note.py`
|
||||
- `scripts/gitea-install-workflow.py`
|
||||
- `.gitea/ISSUE_TEMPLATE/*`
|
||||
- `.gitea/PULL_REQUEST_TEMPLATE.md`
|
||||
- a project-specific `docs/gitea-labels.json`
|
||||
|
||||
Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup.
|
||||
|
||||
## Label Taxonomy
|
||||
|
||||
Use one `type/*` label:
|
||||
|
||||
- `type/bug`
|
||||
- `type/feature`
|
||||
- `type/task`
|
||||
- `type/debt`
|
||||
- `type/docs`
|
||||
|
||||
Use one `status/*` label while the issue is open:
|
||||
|
||||
- `status/triage`: needs ownership, priority, or acceptance criteria.
|
||||
- `status/ready`: ready to implement.
|
||||
- `status/in-progress`: actively being worked.
|
||||
- `status/blocked`: blocked on an external dependency, credential, or decision.
|
||||
- `status/needs-info`: blocked on clarification.
|
||||
|
||||
Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`.
|
||||
|
||||
Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository.
|
||||
|
||||
Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first.
|
||||
|
||||
## Moving TODOs Into Gitea
|
||||
|
||||
Preview inline markers:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-todo-import.py
|
||||
```
|
||||
|
||||
Create missing issues after labels are synced:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-todo-import.py --apply
|
||||
```
|
||||
|
||||
The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items.
|
||||
|
||||
When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference:
|
||||
|
||||
```python
|
||||
# TODO(gitea#123): keep only if the local pointer is still useful
|
||||
```
|
||||
|
||||
Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers.
|
||||
|
||||
For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
It scans repository and product-directory files with backlog-like names, resolves
|
||||
shared labels from the organization label catalogue where available, creates only
|
||||
missing fallback repository labels when needed, imports missing open work, and
|
||||
deduplicates reruns by hidden fingerprint and normalized title.
|
||||
|
||||
## Mirroring Project Docs Into Gitea Wikis
|
||||
|
||||
Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env
|
||||
```
|
||||
|
||||
Apply the wiki mirror:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||
```
|
||||
|
||||
After renaming or deleting repository docs, prune previously managed wiki pages
|
||||
that no longer have a source file:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
|
||||
```
|
||||
|
||||
The default apply path uses the Gitea wiki git repository, not one REST API
|
||||
request per page. It keeps a local checkout cache below
|
||||
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
||||
pushes that commit. This is much faster and avoids REST wiki-page timeouts.
|
||||
Use `--transport api` only when the wiki git remote is unavailable.
|
||||
|
||||
Limit a sync to one repository or one generated page while working:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --apply
|
||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply
|
||||
```
|
||||
|
||||
Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository
|
||||
sync when the set of mirrored pages changes.
|
||||
|
||||
The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related.
|
||||
|
||||
Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages.
|
||||
|
||||
Use the wiki for durable context:
|
||||
|
||||
- project overviews and architecture
|
||||
- workflows, operating notes, and setup references
|
||||
- product concepts, plans, pitches, and whitepapers
|
||||
- historical context that helps interpret issues
|
||||
|
||||
Keep active state in issues:
|
||||
|
||||
- open tasks, TODOs, feature requests, and bugs
|
||||
- priority, blocking status, and acceptance criteria
|
||||
- Codex progress updates and implementation notes
|
||||
|
||||
## Codex State Updates
|
||||
|
||||
Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes:
|
||||
|
||||
- scope understood
|
||||
- files changed
|
||||
- tests or manual checks run
|
||||
- blockers or decisions needed
|
||||
- follow-up issues created
|
||||
|
||||
Preview and post a standardized note:
|
||||
|
||||
```bash
|
||||
./scripts/gitea-codex-note.py \
|
||||
--issue 123 \
|
||||
--status progress \
|
||||
--summary "Implemented capability metadata fallback." \
|
||||
--changed src/govoplan_core/modules/registry.py \
|
||||
--test "./.venv/bin/python -m unittest tests.test_module_system"
|
||||
|
||||
./scripts/gitea-codex-note.py \
|
||||
--issue 123 \
|
||||
--status progress \
|
||||
--summary "Implemented capability metadata fallback." \
|
||||
--changed src/govoplan_core/modules/registry.py \
|
||||
--test "./.venv/bin/python -m unittest tests.test_module_system" \
|
||||
--apply
|
||||
```
|
||||
|
||||
Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded.
|
||||
|
||||
## Ownership Rules
|
||||
|
||||
Create the issue in the repository that owns the change:
|
||||
|
||||
- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components.
|
||||
- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration.
|
||||
- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations.
|
||||
- `govoplan-files`: files-specific backend, frontend, storage, and file workflows.
|
||||
- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior.
|
||||
|
||||
For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details.
|
||||
|
||||
## Cleaning Up Mirrored Sources
|
||||
|
||||
After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state.
|
||||
|
||||
Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding.
|
||||
|
||||
Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path.
|
||||
|
||||
## Docs Versus Issues
|
||||
|
||||
Keep durable facts in docs:
|
||||
|
||||
- architecture and extension points
|
||||
- command references
|
||||
- module boundaries
|
||||
- operational conventions
|
||||
|
||||
Keep changing state in Gitea:
|
||||
|
||||
- TODOs and follow-ups
|
||||
- bugs and feature requests
|
||||
- blocked status
|
||||
- acceptance criteria
|
||||
- implementation notes from active work
|
||||
|
||||
If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history.
|
||||
- `/mnt/DATA/git/govoplan/docs/GITEA_ISSUES.md`
|
||||
|
||||
@@ -458,6 +458,17 @@ routers and live module activation fail fast if two routers register the same
|
||||
HTTP method and path. That keeps OpenAPI output and FastAPI route order from
|
||||
silently masking a module collision.
|
||||
|
||||
Tenant deletion and cleanup use the registry-owned delete-veto contract. A
|
||||
module that owns tenant-bound data may declare `delete_veto_providers` on its
|
||||
manifest for resource types such as `tenant` or `group`. Providers receive
|
||||
`(session, tenant_id, resource_id)` and should return `DeleteVetoIssue`, an
|
||||
iterable of `DeleteVetoIssue`, or `None`; older exception-based providers are
|
||||
still treated as blocking vetoes. Core attributes each issue to the provider
|
||||
module and adds resource context before the tenancy module exposes the issues
|
||||
through the deletion plan. `blocker` issues prevent destructive or retire
|
||||
operations, `warning` issues explain retained data, and `info` issues document
|
||||
non-blocking lifecycle facts.
|
||||
|
||||
## Database And Migrations
|
||||
|
||||
Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest.
|
||||
@@ -470,12 +481,20 @@ Rules:
|
||||
- Keep cross-module foreign-key assumptions explicit and conservative.
|
||||
- Register module metadata in `MigrationSpec` so core can discover it.
|
||||
- Optional module migrations may create multiple Alembic heads. Verification
|
||||
should compare the database heads to the configured script heads instead of
|
||||
assuming one linear revision when multiple modules are enabled.
|
||||
should compare database heads to Alembic's resolved `heads` target instead
|
||||
of assuming one linear revision when multiple modules are enabled. Owner
|
||||
heads can be dependency parents and therefore may not all appear in
|
||||
`alembic_version` after a full-graph upgrade.
|
||||
- Treat migrations as release artifacts. Unreleased migrations may be squashed
|
||||
or rewritten before a stable release; released revision IDs are immutable
|
||||
once an installation may have recorded them. Each stable release records its
|
||||
public migration heads in `docs/migration-release-baselines.json`.
|
||||
- GovOPlaN keeps two Alembic tracks. The default `release` track loads
|
||||
`versions` directories with reviewed release baselines and release-to-release
|
||||
step-up migrations. The explicit `dev` track loads `dev_versions`
|
||||
directories with the detailed development chain. Do not load both tracks for
|
||||
one migration run, and do not switch a database between tracks unless it is a
|
||||
disposable development database.
|
||||
|
||||
## Install, Uninstall, And Catalogs
|
||||
|
||||
@@ -484,16 +503,42 @@ check, maintenance-mode guard, replay state, and installer request queue.
|
||||
Package mutation is performed by `govoplan-module-installer` outside the
|
||||
FastAPI request process.
|
||||
|
||||
Official catalogs can be served as static JSON from `govoplan-web`, but core
|
||||
Official catalogs can be served as static JSON from `addideas-govoplan-website`, but core
|
||||
does not trust the website by location alone. A catalog must pass the configured
|
||||
signature, channel, freshness, and replay rules before a catalog entry can be
|
||||
planned. Catalog entries may declare `license_features`; core checks those
|
||||
against the configured offline license before adding the entry to the install
|
||||
plan.
|
||||
plan. Catalog entries may also declare `migration_safety` as `automatic`,
|
||||
`requires_review`, `forward_only`, or `destructive`; forward-only and
|
||||
destructive entries require explicit operator acknowledgement in the install
|
||||
plan before installer preflight allows activation. Forward-only and destructive
|
||||
catalog entries must also declare a tested recovery path. Catalog update entries
|
||||
can define direct-update windows with `current_version_min` and
|
||||
`current_version_max_exclusive`, mark intermediate `bridge_release` targets, and
|
||||
explicitly opt into reviewed downgrade or same-version package-refresh plans.
|
||||
Module migration order can be declared with `migration_after` and
|
||||
`migration_before` in manifests or release catalogs; installer preflight turns
|
||||
that metadata, module dependencies, and named interface relationships into an
|
||||
ordered migration plan.
|
||||
|
||||
Modules that need live-data work outside Alembic schema revisions may declare
|
||||
`migration_tasks` on `MigrationSpec`. This is deliberately narrower than a
|
||||
general lifecycle hook system. Each task has a stable `task_id`, one of four
|
||||
phases (`pre_migration_check`, `pre_migration_prepare`,
|
||||
`post_migration_backfill`, `post_migration_verify`), a short operator-facing
|
||||
summary, a task version, safety metadata, and an idempotent executor. Installer
|
||||
preflight blocks non-idempotent tasks, forward-only/destructive tasks without
|
||||
operator acknowledgement, and installed manifest tasks that have no executor.
|
||||
Catalog task metadata is surfaced before activation as pending because the
|
||||
executor can only be verified after the package is installed.
|
||||
|
||||
Modules should provide:
|
||||
|
||||
- pinned backend and WebUI package refs for official catalog entries
|
||||
- module dependency metadata for catalog target-state planning
|
||||
- migration-safety metadata for catalog update planning
|
||||
- migration task metadata when live-data checks, preparation, backfills, or
|
||||
verification must run around Alembic
|
||||
- compatibility metadata in the module manifest
|
||||
- named interface contracts in the manifest and catalog entry when the module
|
||||
provides or consumes cross-module APIs
|
||||
@@ -662,7 +707,7 @@ Rules:
|
||||
|
||||
### Dependency Boundary Enforcement
|
||||
|
||||
The repository includes `scripts/check_dependency_boundaries.py`. It enforces the current baseline:
|
||||
The meta repository includes `tools/checks/check_dependency_boundaries.py`. It enforces the current baseline:
|
||||
|
||||
- kernel/core source may not add new direct imports of files/mail/campaign internals
|
||||
- access source may not import files/mail/campaign internals
|
||||
@@ -977,8 +1022,21 @@ The package install-plan API records operator intent only:
|
||||
also reports catalog validity, channel, signature, trust state, and the
|
||||
configured path.
|
||||
- `POST /api/v1/admin/system/modules/install-plan/catalog/{module_id}` saves
|
||||
a planned install row from a validated catalog entry. Catalog signature and
|
||||
approved-channel policy are enforced before the row is saved.
|
||||
a planned install or update row from a validated catalog entry. Installed
|
||||
modules are planned as updates. Catalog signature and approved-channel policy
|
||||
are enforced before the row is saved. When the selected catalog row requires
|
||||
companion dependency or interface-provider updates, the endpoint adds those
|
||||
rows to the plan automatically. The saved plan row can also carry a
|
||||
data-safety acknowledgement used by preflight for forward-only or destructive
|
||||
catalog entries.
|
||||
- Install-plan preflight returns a structured `target_plan` summary so the
|
||||
admin UI can show current version, target version, package refs,
|
||||
migration-safety level, update-window and bridge metadata, recovery metadata,
|
||||
and acknowledgement state without requiring JSON editing.
|
||||
- Install-plan preflight also returns a structured `migration_plan` summary with
|
||||
target enabled modules and ordered module migration steps. When the installer
|
||||
runs with migration enabled, the database migration command receives that
|
||||
target module set and ordered module list.
|
||||
- `POST /api/v1/admin/system/modules/{module_id}/uninstall-plan` saves a
|
||||
planned non-destructive uninstall row for an installed module after it has
|
||||
been disabled. The Python distribution name is resolved from the installed
|
||||
@@ -1158,18 +1216,18 @@ Backend verification from core:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
/mnt/DATA/git/govoplan/.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||
/mnt/DATA/git/govoplan/tools/checks/check_dependency_boundaries.py
|
||||
```
|
||||
|
||||
`scripts/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||
`govoplan/tools/checks/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||
so developer-local npm settings do not create release-check warning noise.
|
||||
|
||||
Focused module contract and permutation verification:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-module-matrix.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-module-matrix.sh
|
||||
```
|
||||
|
||||
Core WebUI host verification:
|
||||
@@ -1183,7 +1241,7 @@ Clean generated `dist`, `.vite`, and source-tree `__pycache__` artifacts after v
|
||||
|
||||
## Release Dependency Rules
|
||||
|
||||
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. Core provides:
|
||||
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. The meta repository provides:
|
||||
|
||||
- `requirements-dev.txt` for local editable backend installs
|
||||
- `requirements-release.txt` for tagged backend module installs
|
||||
|
||||
@@ -16,23 +16,23 @@ resolve modules from tagged git refs or from a package registry.
|
||||
Local development:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Release install from a core checkout plus tagged module repositories:
|
||||
Release install from the meta checkout plus tagged module repositories:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
`.[server]` is resolved relative to the current working directory. If you
|
||||
create the virtualenv elsewhere, still run the install command from the core
|
||||
checkout:
|
||||
`../govoplan-core[server]` is resolved relative to the meta requirements file.
|
||||
If you create the virtualenv elsewhere, still run the install command from the
|
||||
meta checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
||||
```
|
||||
|
||||
@@ -64,9 +64,9 @@ referenced there exist, generate the committed release lockfile without
|
||||
touching the development package files:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/generate-release-lock.sh
|
||||
cd webui
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/release/generate-release-lock.sh
|
||||
cd /mnt/DATA/git/govoplan-core/webui
|
||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
||||
```
|
||||
|
||||
@@ -96,25 +96,27 @@ generated in a clean release workspace from tagged git dependencies.
|
||||
|
||||
## Release Tag Script
|
||||
|
||||
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps
|
||||
The normal release path is automated by `/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh`: it bumps
|
||||
or accepts the target version, updates Python/WebUI/module manifest versions,
|
||||
commits/tags/pushes the module repositories first, regenerates
|
||||
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
||||
working tree has already been bumped, pass the current version explicitly:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/push-release-tag.sh --version 0.1.6
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/release/push-release-tag.sh --version 0.1.6
|
||||
```
|
||||
|
||||
`scripts/generate-release-catalog.py` reads installed/discovered
|
||||
`/mnt/DATA/git/govoplan/tools/release/generate-release-catalog.py` reads installed/discovered
|
||||
`ModuleManifest` objects while writing catalog entries. When a manifest is
|
||||
available, the catalog entry uses the manifest version, points package refs at
|
||||
`v<manifest.version>`, and copies `provides_interfaces` /
|
||||
`requires_interfaces` from the manifest. If a manifest cannot be discovered,
|
||||
the entry falls back to the release version passed with `--version` and omits
|
||||
interface metadata. This keeps the catalog aligned with independently
|
||||
versioned module packages instead of relying on a hardcoded compatibility table.
|
||||
`requires_interfaces` from the manifest. It also copies module migration order
|
||||
and `migration_tasks` metadata when present. If a manifest cannot be
|
||||
discovered, the entry falls back to the release version passed with `--version`
|
||||
and omits interface and migration-task metadata. This keeps the catalog aligned
|
||||
with independently versioned module packages instead of relying on a hardcoded
|
||||
compatibility table.
|
||||
|
||||
The script also includes GovOPlaN roadmap/scaffold module repositories that do
|
||||
not yet have package metadata. Those repositories are committed, tagged, and
|
||||
@@ -155,8 +157,8 @@ running server may plan and validate package changes, but package mutation is
|
||||
performed by the separate installer daemon or an operator shell during
|
||||
maintenance mode.
|
||||
|
||||
`govoplan-web` is the public static distribution surface for official catalog
|
||||
resources:
|
||||
`addideas-govoplan-website` is the public static distribution surface for
|
||||
official catalog resources:
|
||||
|
||||
- signed module package catalogs, grouped by release channel
|
||||
- public catalog keyrings
|
||||
@@ -205,6 +207,27 @@ Each module entry can declare:
|
||||
- WebUI package name and pinned install reference
|
||||
- display metadata and tags
|
||||
- `license_features`, the feature entitlements required to plan that install
|
||||
- `dependencies` and `optional_dependencies`, the module ids expected in the
|
||||
target module set
|
||||
- `migration_safety`, one of `automatic`, `requires_review`, `forward_only`,
|
||||
or `destructive`
|
||||
- `migration_notes`, operator-facing data/migration guidance for review,
|
||||
forward-only, or destructive changes
|
||||
- `migration_after` and `migration_before`, explicit module ids used to order
|
||||
module-owned migration heads when a release needs a live-data sequencing rule
|
||||
- `migration_tasks`, constrained live-data tasks that run around Alembic
|
||||
migration phases. Each task declares `task_id`, `phase`, `summary`,
|
||||
`task_version`, `safety`, `idempotent`, and optionally `timeout_seconds`.
|
||||
The allowed phases are `pre_migration_check`, `pre_migration_prepare`,
|
||||
`post_migration_backfill`, and `post_migration_verify`.
|
||||
- `current_version_min` and `current_version_max_exclusive`, the installed
|
||||
version window from which this catalog target may be applied directly
|
||||
- `bridge_release` and `bridge_notes`, marking a target as an intermediate
|
||||
compatibility release in a staged update path
|
||||
- `allow_downgrade` and `allow_same_version`, explicit opt-ins for reviewed
|
||||
rollback or package-refresh plans
|
||||
- `recovery_tested` and `recovery_notes`, documenting the rehearsal for
|
||||
forward-only or destructive data changes
|
||||
- `provides_interfaces`, named interface contracts exported by this module
|
||||
- `requires_interfaces`, named interface contracts and version ranges required
|
||||
by this module
|
||||
@@ -264,16 +287,32 @@ source/path, source type, cache path, channel, sequence, generated/validity
|
||||
timestamps, signature state, trusted key id, and cache state where available.
|
||||
Catalog provenance changes preflight severity:
|
||||
|
||||
- catalog-sourced installs require a configured, valid package catalog before
|
||||
activation
|
||||
- catalog-sourced installs and updates require a configured, valid package
|
||||
catalog before activation
|
||||
- invalid, untrusted, expired, not-yet-valid, replayed, or unapproved-channel
|
||||
catalogs block catalog-sourced installs
|
||||
catalogs block catalog-sourced installs and updates
|
||||
- the same catalog validation failures remain warnings for manual install
|
||||
plans, so operators can still use offline or emergency package refs
|
||||
- valid-catalog warnings, such as intentionally unsigned local catalogs when
|
||||
signature enforcement is disabled, remain warnings
|
||||
- selected catalog entries with unsatisfied non-optional named interface ranges
|
||||
block activation before the installer runs
|
||||
- selected catalog entries whose target dependencies are neither installed nor
|
||||
planned block activation before the installer runs
|
||||
- catalog update targets older than the installed module version block unless
|
||||
the catalog entry declares `allow_downgrade: true`
|
||||
- catalog update targets equal to the installed module version block unless the
|
||||
catalog entry declares `allow_same_version: true`
|
||||
- catalog update targets with a `current_version_min` /
|
||||
`current_version_max_exclusive` window block when the installed version is
|
||||
outside that window; publish and apply a bridge release instead
|
||||
- catalog entries marked `forward_only` or `destructive` block activation until
|
||||
the plan row has an explicit data-safety acknowledgement
|
||||
- catalog entries marked `forward_only` or `destructive` also block unless the
|
||||
catalog entry declares `recovery_tested: true` and either the catalog entry or
|
||||
operator plan row contains recovery notes
|
||||
- catalog entries marked `destructive` also require catalog migration notes or
|
||||
operator notes describing the cleanup or retirement plan
|
||||
|
||||
### Update Paths
|
||||
|
||||
@@ -283,6 +322,41 @@ catalog validation snapshot. The installer may install multiple packages into
|
||||
the environment before activation, then validate the discovered manifests and
|
||||
activate the resulting set together.
|
||||
|
||||
Install-plan rows support explicit `install`, `update`, and `uninstall`
|
||||
actions. Catalog planning writes `update` when the module is already installed.
|
||||
Preflight resolves the target set from installed manifests plus the planned
|
||||
catalog entries. Unplanned catalog entries are not treated as installed. When a
|
||||
catalog entry would satisfy a missing dependency or named interface, preflight
|
||||
blocks activation with a companion-update issue; the admin catalog planner adds
|
||||
those companion rows automatically when it can resolve them from the current
|
||||
catalog. The preflight response also includes a structured `target_plan` summary
|
||||
with each planned module's action, current version, catalog target version,
|
||||
package refs, migration-safety level, current-version update window, bridge
|
||||
metadata, recovery metadata, and acknowledgement state.
|
||||
|
||||
Database migrations are planned against that same target module set. When the
|
||||
installer is run with `--migrate`, it calls `govoplan_core.commands.init_db`
|
||||
with the target enabled modules rather than the pre-update startup module list,
|
||||
so newly installed module migration directories are discovered before
|
||||
activation. Preflight also returns a structured migration plan. Its step order is
|
||||
derived from:
|
||||
|
||||
- manifest and catalog `migration_after` / `migration_before` declarations
|
||||
- module dependencies and optional dependencies when both modules are in the
|
||||
target plan
|
||||
- named interface provider/consumer relationships when both sides are in the
|
||||
target plan
|
||||
|
||||
Preflight blocks cycles in that ordering graph. It also blocks non-idempotent
|
||||
module migration tasks, forward-only/destructive tasks without operator
|
||||
acknowledgement, and installed manifest tasks that declare no executor.
|
||||
Catalog-only task executors are marked as pending because they can only be
|
||||
confirmed after the target package is installed. The migrator runs pre-migration
|
||||
tasks, upgrades the ordered module heads first, finishes with Alembic `heads`,
|
||||
and then runs post-migration tasks, so Alembic's revision graph remains
|
||||
authoritative while GovOPlaN still gives operators a module-aware live-data
|
||||
order.
|
||||
|
||||
This avoids circular "upgrade A first / upgrade B first" traps: named interface
|
||||
requirements are solved against the target set, not against each intermediate
|
||||
package-install moment. If the target set cannot satisfy all non-optional
|
||||
@@ -303,11 +377,25 @@ Live data upgrades need an even stricter rule:
|
||||
must publish an intermediate compatibility release rather than a circular
|
||||
update chain
|
||||
|
||||
The release catalog is the first safety gate for this. Generated catalogs mark
|
||||
modules with registered migrations as `requires_review` by default. Release
|
||||
authors should keep that value for ordinary reversible migrations, raise it to
|
||||
`forward_only` when database rollback requires restoring a snapshot, and raise
|
||||
it to `destructive` when the update removes or irreversibly rewrites persisted
|
||||
data. Forward-only and destructive entries must include `recovery_tested: true`
|
||||
and recovery notes after a verified restore or forward-recovery rehearsal. The
|
||||
admin install-plan UI exposes the safety level and lets operators record an
|
||||
explicit acknowledgement; preflight keeps acknowledged forward-only/destructive
|
||||
changes visible as warnings.
|
||||
|
||||
In practice, circular dependencies are avoided by designing interfaces with
|
||||
compatibility windows and by publishing bridge releases. A bridge release keeps
|
||||
the old interface while introducing the new one, allowing dependent modules to
|
||||
move first; a later release can retire the old interface after every dependent
|
||||
module has a compatible target version.
|
||||
module has a compatible target version. Use `current_version_min` and
|
||||
`current_version_max_exclusive` to make those direct-update windows explicit in
|
||||
the catalog, and set `bridge_release: true` on intermediate targets that exist
|
||||
primarily to carry installations safely across a compatibility gap.
|
||||
|
||||
Trusted catalog keys are configured locally:
|
||||
|
||||
@@ -335,11 +423,11 @@ Dependency vulnerability checks are documented in
|
||||
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same
|
||||
The Gitea workflow in `govoplan/.gitea/workflows/dependency-audit.yml` runs the same
|
||||
check against release dependency refs on pushes, pull requests, and a weekly
|
||||
schedule.
|
||||
|
||||
@@ -499,22 +587,23 @@ entitlements without changing the source license of the repositories.
|
||||
|
||||
Production-grade distribution still needs remote registry/git artifact
|
||||
resolution before package-manager apply, a hardened catalog publishing pipeline
|
||||
in `govoplan-web`, and automated key rotation and emergency revocation drills.
|
||||
that writes to `addideas-govoplan-website`, and automated key rotation and
|
||||
emergency revocation drills.
|
||||
|
||||
## Release Catalog Publishing
|
||||
|
||||
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
||||
verified by `govoplan-core` before installer plans are accepted. Private signing
|
||||
keys must stay outside all git repositories. Public keyrings are published with
|
||||
the website.
|
||||
GovOPlaN release catalogs are published to `addideas-govoplan-website` as
|
||||
static JSON and verified by `govoplan-core` before installer plans are
|
||||
accepted. Private signing keys must stay outside all git repositories. Public
|
||||
keyrings are published with the website.
|
||||
|
||||
Create the first catalog signing key on the release machine:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
mkdir -p "$KEY_DIR"
|
||||
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
||||
./.venv/bin/python tools/release/generate-catalog-keypair.py \
|
||||
--key-id release-key-1 \
|
||||
--private-key "$KEY_DIR/release-key-1.pem" \
|
||||
--public-key "$KEY_DIR/release-key-1.pub" \
|
||||
@@ -524,12 +613,12 @@ mkdir -p "$KEY_DIR"
|
||||
Keep `release-key-1.pem` private. The generated keyring contains only public
|
||||
material.
|
||||
|
||||
Generate the signed catalog into `govoplan-web`:
|
||||
Generate the signed catalog into `addideas-govoplan-website`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/publish-release-catalog.sh \
|
||||
tools/release/publish-release-catalog.sh \
|
||||
--version <x.y.z> \
|
||||
--sequence 202607071340 \
|
||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||
@@ -538,8 +627,8 @@ scripts/publish-release-catalog.sh \
|
||||
|
||||
This writes:
|
||||
|
||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
||||
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/channels/stable.json`
|
||||
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/keyring.json`
|
||||
|
||||
The wrapper validates the catalog with core using the generated public keyring.
|
||||
|
||||
@@ -547,14 +636,14 @@ For normal module/core releases, first audit and record migration baselines,
|
||||
then tag and push the module/core repos. Finally publish the website catalog:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||
```
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/publish-release-catalog.sh \
|
||||
tools/release/publish-release-catalog.sh \
|
||||
--version <x.y.z> \
|
||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||
--build-web \
|
||||
@@ -575,16 +664,16 @@ The public keyring URL is:
|
||||
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
||||
```
|
||||
|
||||
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
||||
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` can publish the web catalog after module and core
|
||||
tags have been pushed. It runs the migration release audit in automatic mode:
|
||||
warning-only before the first recorded migration baseline, strict after a
|
||||
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
||||
mode explicitly:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||
scripts/push-release-tag.sh \
|
||||
tools/release/push-release-tag.sh \
|
||||
--bump subversion \
|
||||
--strict-migration-audit \
|
||||
--publish-web-catalog \
|
||||
@@ -596,6 +685,50 @@ Use `--catalog-signing-key` more than once during a key rotation window. The
|
||||
catalog will contain multiple signatures and the public keyring will include the
|
||||
corresponding public keys.
|
||||
|
||||
## Release Doctor
|
||||
|
||||
Use `/mnt/DATA/git/govoplan/tools/release/release-doctor.py` before release preparation and again before
|
||||
publishing. It inspects repository state, local versions, release tags,
|
||||
migration audit state, release package refs, stable catalog/keyring state, and
|
||||
optionally public catalog availability. The default mode is read-only and
|
||||
offline. Status output is intentionally concise:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z>
|
||||
```
|
||||
|
||||
Use `--details` when the full findings should be printed directly:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z> --details
|
||||
```
|
||||
|
||||
For concise guidance, print only suggested next commands:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py next --target-version <x.y.z>
|
||||
```
|
||||
|
||||
For an operator-guided session, run interactive mode. The doctor asks which
|
||||
suggested command to run; mutating commands require typing `RUN` before they
|
||||
execute:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-doctor.py interactive --target-version <x.y.z>
|
||||
```
|
||||
|
||||
Interactive mode starts with a compact summary and waits for an action. It can
|
||||
open the full report in the configured pager, run one suggested command, repair
|
||||
Git `safe.directory` dubious-ownership blocks after explicit confirmation, push
|
||||
all clean repositories that are ahead of their upstream, or commit and push all
|
||||
dirty repositories. The dirty-repository bulk action asks for a commit message,
|
||||
skips repositories that are behind upstream, and requires typing
|
||||
`COMMIT AND PUSH` before it stages, commits, and pushes.
|
||||
|
||||
Add `--online` when remote tag and public catalog/keyring reachability should be
|
||||
checked. Add `--json` when a CI job or another tool should consume the report.
|
||||
|
||||
On a GovOPlaN installation that should consume the official stable catalog:
|
||||
|
||||
```bash
|
||||
@@ -619,7 +752,7 @@ operator-supervised package update with restart and health checks.
|
||||
Key rotation for published catalogs:
|
||||
|
||||
1. Generate the next private key outside git.
|
||||
2. Run `publish-release-catalog.sh` with both signing keys.
|
||||
2. Run `/mnt/DATA/git/govoplan/tools/release/publish-release-catalog.sh` with both signing keys.
|
||||
3. Publish the web catalog/keyring.
|
||||
4. Roll the new public keyring into installations.
|
||||
5. Stop signing with the old key after the supported fleet trusts the new key.
|
||||
@@ -633,15 +766,15 @@ smoke check before tagging or publishing catalogs. Start the local testbed,
|
||||
then run the permutation check from the core checkout:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
set -a
|
||||
. dev/postgres/.env
|
||||
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||
set +a
|
||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||
tools/checks/postgres-integration-check.py \
|
||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||
--reset-schema
|
||||
```
|
||||
@@ -654,56 +787,94 @@ throwaway database.
|
||||
## Migration Baselines
|
||||
|
||||
Development migrations may be small and numerous while a feature is moving.
|
||||
Before a stable release, unreleased migrations may be rewritten or squashed into
|
||||
a release-level baseline or release-to-release upgrade migration. After a
|
||||
release tag has shipped, released migration revision IDs are immutable.
|
||||
GovOPlaN keeps those detailed migrations on an explicit development track and
|
||||
publishes reviewed release shortcuts on the release track. Before a stable
|
||||
release, unreleased development migrations may be squashed into a release-level
|
||||
baseline or release-to-release upgrade migration. After a release tag has
|
||||
shipped, released migration revision IDs are immutable.
|
||||
|
||||
The release policy is:
|
||||
|
||||
- unreleased migrations may be folded before release;
|
||||
- released migrations are never rewritten or deleted;
|
||||
- unreleased development migrations live in `dev_versions` and are not deleted
|
||||
when a release shortcut is added;
|
||||
- released migrations live in `versions` and are never rewritten or deleted;
|
||||
- future release shortcuts are additive. A new release-to-release migration
|
||||
starts from the previous recorded release heads, so installations can upgrade
|
||||
through sane release steps instead of replaying every development revision;
|
||||
- each stable release records the public migration head revisions in
|
||||
`docs/migration-release-baselines.json`;
|
||||
- `heads` records the Alembic dependency-leaf graph heads for a full graph,
|
||||
while `owner_heads` records the latest migration revision per migration
|
||||
owner for subset installs and review;
|
||||
- fresh installations should apply release-level baselines/upgrades, not
|
||||
unreleased create-then-rename churn;
|
||||
- release-to-release schema changes should be folded into one reviewed
|
||||
migration per migration owner where practical.
|
||||
|
||||
The default runtime track is `release`:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_MIGRATION_TRACK=release ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||
```
|
||||
|
||||
Use `dev` only for local/disposable development databases that intentionally
|
||||
need the detailed chain:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_MIGRATION_TRACK=dev ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||
```
|
||||
|
||||
Production, release checks, operator install flows, and the normal development
|
||||
launcher should stay on the `release` track unless a fresh/disposable database
|
||||
is intentionally being used to test the detailed development chain.
|
||||
|
||||
Audit the current graph during release preparation:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
./.venv/bin/python scripts/release-migration-audit.py
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py
|
||||
```
|
||||
|
||||
Audit the detailed development graph separately when a squash is prepared:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --track dev
|
||||
```
|
||||
|
||||
Generate the reviewed/manual squash checklist:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --squash-plan
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --squash-plan
|
||||
```
|
||||
|
||||
After the release migrations have been reviewed and the graph is final, record
|
||||
the release baseline:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --record-release <x.y.z>
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --record-release <x.y.z>
|
||||
```
|
||||
|
||||
Use strict mode to verify that the current heads are recorded:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||
```
|
||||
|
||||
`scripts/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||
non-strict while no release baseline exists, strict after the first baseline is
|
||||
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
||||
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
||||
only for emergency/manual release work.
|
||||
|
||||
Before the first stable release, fold the current development chain into the
|
||||
first public baseline and record that baseline in
|
||||
`docs/migration-release-baselines.json`. The tracking issue is
|
||||
The first public baseline is v0.1.7. It intentionally adds release-track
|
||||
shortcuts for the unreleased v0.0.0 -> v0.1.7 development chains while keeping
|
||||
the detailed chains on the `dev` track. No production installations existed
|
||||
before that baseline, so pre-v0.1.7 development revisions are not release
|
||||
upgrade targets. Future release-to-release changes must start from a recorded
|
||||
release baseline and add a new release-track step-up instead of replacing prior
|
||||
release shortcuts. The tracking issue is
|
||||
`add-ideas/govoplan-core#223`.
|
||||
|
||||
## Related Operator Documents
|
||||
@@ -719,11 +890,11 @@ first public baseline and record that baseline in
|
||||
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
||||
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
||||
calendar, and scaffold module repositories together.
|
||||
- Update `requirements-release.txt` and `webui/package.release.json` when the
|
||||
- Update meta `requirements-release.txt` and core `webui/package.release.json` when the
|
||||
release tag changes.
|
||||
- Generate the committed full-product release lockfile from
|
||||
`package.release.json` with `scripts/generate-release-lock.sh`.
|
||||
- Run `scripts/release-migration-audit.py --strict` after recording a release
|
||||
`package.release.json` with `/mnt/DATA/git/govoplan/tools/release/generate-release-lock.sh`.
|
||||
- Run `/mnt/DATA/git/govoplan/tools/release/release-migration-audit.py --strict` after recording a release
|
||||
baseline.
|
||||
- Run the PostgreSQL release check against a disposable database.
|
||||
- Publish the signed catalog through the release catalog publishing flow above.
|
||||
|
||||
15
docs/SECURITY_AUDIT.md
Normal file
15
docs/SECURITY_AUDIT.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Security Audit Toolchain
|
||||
|
||||
The shared GovOPlaN security audit toolbox moved to the meta repository.
|
||||
|
||||
Use:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||
```
|
||||
|
||||
Canonical documentation:
|
||||
|
||||
- `/mnt/DATA/git/govoplan/docs/SECURITY_AUDIT.md`
|
||||
@@ -22,7 +22,7 @@ daemon. The API server must not run package managers from request handlers.
|
||||
Generate a self-hosted template:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
cd /mnt/DATA/git/govoplan
|
||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||
--profile self-hosted \
|
||||
--generate-secrets \
|
||||
@@ -49,31 +49,31 @@ catalog source is configured.
|
||||
Use the local production-like wrapper for repeatable rehearsal:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
scripts/production-like-dev.sh validate-config
|
||||
scripts/production-like-dev.sh seed
|
||||
scripts/production-like-dev.sh start
|
||||
cd /mnt/DATA/git/govoplan
|
||||
tools/launch/production-like-dev.sh validate-config
|
||||
tools/launch/production-like-dev.sh seed
|
||||
tools/launch/production-like-dev.sh start
|
||||
```
|
||||
|
||||
Stop Docker dependencies:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh stop
|
||||
tools/launch/production-like-dev.sh stop
|
||||
```
|
||||
|
||||
Reset all profile data:
|
||||
|
||||
```bash
|
||||
scripts/production-like-dev.sh reset --yes
|
||||
tools/launch/production-like-dev.sh reset --yes
|
||||
```
|
||||
|
||||
The start command delegates to `scripts/launch-production-like-dev.sh`, which
|
||||
The start command delegates to `tools/launch/launch-production-like-dev.sh`, which
|
||||
runs API, worker, and WebUI in the foreground. Stop those processes with
|
||||
`Ctrl+C` in the launcher terminal.
|
||||
|
||||
## Module Boundary Gate
|
||||
|
||||
`scripts/check_dependency_boundaries.py` is part of the focused verification
|
||||
`govoplan/tools/checks/check_dependency_boundaries.py` is part of the focused verification
|
||||
path. It checks backend imports and WebUI package/source imports so modules do
|
||||
not grow hidden runtime dependencies on each other. Feature modules should
|
||||
integrate through core capabilities, backend APIs/events, route contributions,
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
Commands:
|
||||
|
||||
```bash
|
||||
cd /mnt/DATA/git/govoplan-core
|
||||
bash scripts/check-dependency-audits.sh
|
||||
cd /mnt/DATA/git/govoplan
|
||||
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-dependency-audits.sh
|
||||
```
|
||||
|
||||
Status: remediated.
|
||||
@@ -50,10 +50,10 @@ Remediation applied on 2026-07-09:
|
||||
|
||||
Post-remediation result:
|
||||
|
||||
- `bash scripts/check-dependency-audits.sh`: passed, no known Python
|
||||
- `bash tools/checks/check-dependency-audits.sh`: passed, no known Python
|
||||
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
||||
- `python -m pip check`: passed.
|
||||
- `bash scripts/check-module-matrix.sh`: passed.
|
||||
- `bash tools/checks/check-module-matrix.sh`: passed.
|
||||
- `python -m unittest tests.test_api_smoke`: passed.
|
||||
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
||||
|
||||
|
||||
@@ -1,416 +0,0 @@
|
||||
[
|
||||
{
|
||||
"name": "type/bug",
|
||||
"color": "d73a4a",
|
||||
"description": "A reproducible defect, regression, or incorrect behavior.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/feature",
|
||||
"color": "0e8a16",
|
||||
"description": "New user-visible behavior or platform capability.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/task",
|
||||
"color": "1d76db",
|
||||
"description": "Implementation, maintenance, migration, or operational work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/debt",
|
||||
"color": "fbca04",
|
||||
"description": "Cleanup, refactoring, risk reduction, or deferred engineering work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "type/docs",
|
||||
"color": "5319e7",
|
||||
"description": "Documentation, process, or developer workflow work.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p0",
|
||||
"color": "b60205",
|
||||
"description": "Immediate stop-the-line priority.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p1",
|
||||
"color": "d93f0b",
|
||||
"description": "High priority for the next focused work window.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p2",
|
||||
"color": "fbca04",
|
||||
"description": "Normal planned priority.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "priority/p3",
|
||||
"color": "c2e0c6",
|
||||
"description": "Low priority or opportunistic cleanup.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/triage",
|
||||
"color": "d4c5f9",
|
||||
"description": "Needs review, ownership, priority, or acceptance criteria.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/ready",
|
||||
"color": "0e8a16",
|
||||
"description": "Ready for implementation.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/in-progress",
|
||||
"color": "0052cc",
|
||||
"description": "Currently being worked.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/blocked",
|
||||
"color": "b60205",
|
||||
"description": "Cannot progress without a decision, dependency, credential, or external change.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "status/needs-info",
|
||||
"color": "f9d0c4",
|
||||
"description": "Needs clarifying input before implementation can proceed safely.",
|
||||
"exclusive": true
|
||||
},
|
||||
{
|
||||
"name": "module/access",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/addresses",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Addresses module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/admin",
|
||||
"color": "006b75",
|
||||
"description": "GovOPlaN Admin module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/appointments",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN Appointments module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/audit",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN Audit module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/calendar",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Calendar module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/campaign",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN campaign module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/cases",
|
||||
"color": "1d76db",
|
||||
"description": "GovOPlaN Cases module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/connectors",
|
||||
"color": "c2e0c6",
|
||||
"description": "GovOPlaN Connectors module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/core",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN core runner, shared primitives, shell, or extension points.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/dms",
|
||||
"color": "c5def5",
|
||||
"description": "GovOPlaN Dms module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/erp",
|
||||
"color": "fef2c0",
|
||||
"description": "GovOPlaN Erp module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/files",
|
||||
"color": "006b75",
|
||||
"description": "GovOPlaN files module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/fit-connect",
|
||||
"color": "fbca04",
|
||||
"description": "GovOPlaN Fit Connect module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/forms",
|
||||
"color": "f9d0c4",
|
||||
"description": "GovOPlaN Forms module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/identity-trust",
|
||||
"color": "d4c5f9",
|
||||
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/identity",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Identity module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/idm",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN Idm module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/ledger",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Ledger module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/mail",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN mail module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/notifications",
|
||||
"color": "d876e3",
|
||||
"description": "GovOPlaN Notifications module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/ops",
|
||||
"color": "0e8a16",
|
||||
"description": "GovOPlaN Ops module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/organizations",
|
||||
"color": "bfdadc",
|
||||
"description": "GovOPlaN Organizations module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/payments",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN Payments module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/policy",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Policy module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/portal",
|
||||
"color": "1d76db",
|
||||
"description": "GovOPlaN Portal module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/postbox",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Postbox module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/reporting",
|
||||
"color": "c2e0c6",
|
||||
"description": "GovOPlaN Reporting module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/search",
|
||||
"color": "bfdadc",
|
||||
"description": "GovOPlaN Search module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/scheduling",
|
||||
"color": "d93f0b",
|
||||
"description": "GovOPlaN Scheduling module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/tasks",
|
||||
"color": "c5def5",
|
||||
"description": "GovOPlaN Tasks module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/templates",
|
||||
"color": "fef2c0",
|
||||
"description": "GovOPlaN Templates module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/tenancy",
|
||||
"color": "e4e669",
|
||||
"description": "GovOPlaN Tenancy module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/web",
|
||||
"color": "bfd4f2",
|
||||
"description": "GovOPlaN public website, product page, or publication content.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/workflow",
|
||||
"color": "f9d0c4",
|
||||
"description": "GovOPlaN Workflow module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xoev",
|
||||
"color": "d4c5f9",
|
||||
"description": "GovOPlaN Xoev module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xrechnung",
|
||||
"color": "0052cc",
|
||||
"description": "GovOPlaN Xrechnung module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "module/xta-osci",
|
||||
"color": "5319e7",
|
||||
"description": "GovOPlaN Xta Osci module behavior or integration.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/auth",
|
||||
"color": "bfd4f2",
|
||||
"description": "Authentication, sessions, access bootstrap, or login behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/tenancy",
|
||||
"color": "bfdadc",
|
||||
"description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/rbac",
|
||||
"color": "c5def5",
|
||||
"description": "Permissions, roles, delegation, or authorization policy.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/governance",
|
||||
"color": "fef2c0",
|
||||
"description": "Governance policy, audit, privacy, retention, or compliance behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/module-system",
|
||||
"color": "bfe5bf",
|
||||
"description": "Module discovery, manifests, capabilities, routing, or optional integrations.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/migrations",
|
||||
"color": "e4e669",
|
||||
"description": "Alembic migrations, schema bootstrap, or persistence evolution.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/webui",
|
||||
"color": "1d76db",
|
||||
"description": "Shared WebUI shell, frontend components, routing, or frontend tests.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/api",
|
||||
"color": "5319e7",
|
||||
"description": "HTTP API contracts, routers, schemas, or API smoke behavior.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/db",
|
||||
"color": "0e8a16",
|
||||
"description": "Database sessions, models, transactions, or persistence primitives.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/devex",
|
||||
"color": "bfd4f2",
|
||||
"description": "Local developer workflow, scripts, tests, tooling, or release helpers.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/release",
|
||||
"color": "c2e0c6",
|
||||
"description": "Versioning, release locks, tags, packaging, or dependency pins.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/docs",
|
||||
"color": "5319e7",
|
||||
"description": "Durable documentation and project guidance.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "area/marketing",
|
||||
"color": "f9d0c4",
|
||||
"description": "Public website, product messaging, publication copy, or legal page content.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "source/todo-scan",
|
||||
"color": "cccccc",
|
||||
"description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "source/backlog-import",
|
||||
"color": "bfdadc",
|
||||
"description": "Imported from markdown backlog, roadmap, plan, or TODO files.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "codex/ready",
|
||||
"color": "0e8a16",
|
||||
"description": "Suitable for Codex to pick up with the existing issue context.",
|
||||
"exclusive": false
|
||||
},
|
||||
{
|
||||
"name": "codex/needs-human",
|
||||
"color": "f9d0c4",
|
||||
"description": "Needs an explicit human decision before Codex should implement.",
|
||||
"exclusive": false
|
||||
}
|
||||
]
|
||||
@@ -1,4 +1,89 @@
|
||||
{
|
||||
"version": 1,
|
||||
"releases": []
|
||||
"releases": [
|
||||
{
|
||||
"heads": [
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revision": "2c3d4e5f7081"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revision": "3d4e5f708192"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revision": "8f9a0b1c2d3e"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revision": "9e0f1a2b3c4d"
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revision": "a7b8c9d0e1f3"
|
||||
}
|
||||
],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": "govoplan-access",
|
||||
"revisions": [
|
||||
"4a5b6c7d8e9f"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-calendar",
|
||||
"revisions": [
|
||||
"9e0f1a2b3c4d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-campaign",
|
||||
"revisions": [
|
||||
"2c3d4e5f7081"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-core",
|
||||
"revisions": [
|
||||
"4f2a9c8e7b6d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-files",
|
||||
"revisions": [
|
||||
"a7b8c9d0e1f3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-identity",
|
||||
"revisions": [
|
||||
"5c6d7e8f9a10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-idm",
|
||||
"revisions": [
|
||||
"8f9a0b1c2d3e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-mail",
|
||||
"revisions": [
|
||||
"3d4e5f708192"
|
||||
]
|
||||
},
|
||||
{
|
||||
"owner": "govoplan-organizations",
|
||||
"revisions": [
|
||||
"6d7e8f9a0b1c"
|
||||
]
|
||||
}
|
||||
],
|
||||
"recorded_at": "2026-07-11T01:39:45Z",
|
||||
"release": "0.1.7",
|
||||
"track": "release",
|
||||
"squash_policy": "reviewed-manual"
|
||||
}
|
||||
],
|
||||
"version": 1
|
||||
}
|
||||
|
||||
@@ -15,6 +15,29 @@
|
||||
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||
"webui_package": "@govoplan/files-webui",
|
||||
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||
"migration_safety": "forward_only",
|
||||
"migration_notes": "Database rollback requires restoring the pre-update snapshot.",
|
||||
"migration_after": ["access"],
|
||||
"migration_before": ["campaigns"],
|
||||
"migration_tasks": [
|
||||
{
|
||||
"task_id": "backfill_spaces",
|
||||
"phase": "post_migration_backfill",
|
||||
"summary": "Backfill file spaces after the schema migration.",
|
||||
"task_version": "1",
|
||||
"safety": "forward_only",
|
||||
"idempotent": true,
|
||||
"timeout_seconds": 300
|
||||
}
|
||||
],
|
||||
"current_version_min": "0.1.0",
|
||||
"current_version_max_exclusive": "0.2.0",
|
||||
"bridge_release": true,
|
||||
"bridge_notes": "Keeps the 0.1.x campaign attachment interface while introducing the 0.2.x contract.",
|
||||
"allow_downgrade": false,
|
||||
"allow_same_version": false,
|
||||
"recovery_tested": true,
|
||||
"recovery_notes": "Snapshot restore and forward recovery were rehearsed on the release candidate dataset.",
|
||||
"provides_interfaces": [
|
||||
{
|
||||
"name": "files.campaign_attachments",
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "govoplan-core"
|
||||
version = "0.1.6"
|
||||
version = "0.1.8"
|
||||
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
-e .[server]
|
||||
-e ../govoplan-tenancy
|
||||
-e ../govoplan-organizations
|
||||
-e ../govoplan-identity
|
||||
-e ../govoplan-access
|
||||
-e ../govoplan-admin
|
||||
-e ../govoplan-policy
|
||||
-e ../govoplan-audit
|
||||
-e ../govoplan-dashboard
|
||||
-e ../govoplan-files
|
||||
-e ../govoplan-mail
|
||||
-e ../govoplan-campaign
|
||||
-e ../govoplan-calendar
|
||||
-e ../govoplan-docs
|
||||
-e ../govoplan-ops
|
||||
httpx==0.28.1
|
||||
httpx2>=2.5,<3
|
||||
pip-audit>=2.9,<3
|
||||
@@ -1,17 +0,0 @@
|
||||
# Release install from tagged module repositories.
|
||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
||||
.[server]
|
||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
|
||||
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
|
||||
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
|
||||
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
|
||||
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
|
||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
|
||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
|
||||
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
|
||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
|
||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
|
||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
|
||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
|
||||
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
|
||||
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
||||
|
||||
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
||||
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
||||
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
python_status=0
|
||||
npm_status=0
|
||||
|
||||
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
||||
|
||||
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
||||
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
if importlib.util.find_spec("pip") is None:
|
||||
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
||||
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
||||
print(" python -m venv .venv", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
||||
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
||||
raise SystemExit(127)
|
||||
PY
|
||||
|
||||
"$PYTHON" -m pip check
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
prefix = pathlib.Path(sys.prefix)
|
||||
legacy_patterns = (
|
||||
"__editable__.govoplan_module_multimailer-*.pth",
|
||||
"__editable___govoplan_module_multimailer_*_finder.py",
|
||||
"govoplan_module_multimailer-*.dist-info",
|
||||
)
|
||||
problems: list[pathlib.Path] = []
|
||||
|
||||
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
||||
for pattern in legacy_patterns:
|
||||
problems.extend(site_packages.glob(pattern))
|
||||
|
||||
for dist in importlib.metadata.distributions():
|
||||
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
||||
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
||||
|
||||
if problems:
|
||||
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
||||
for path in sorted(set(problems)):
|
||||
print(f" {path}", file=sys.stderr)
|
||||
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("No stale legacy multimailer package metadata found.")
|
||||
PY
|
||||
|
||||
"$PYTHON" - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path.cwd()
|
||||
scan_roots = [
|
||||
root / "src",
|
||||
root / "tests",
|
||||
root.parent / "govoplan-access" / "src",
|
||||
root.parent / "govoplan-admin" / "src",
|
||||
root.parent / "govoplan-audit" / "src",
|
||||
root.parent / "govoplan-calendar" / "src",
|
||||
root.parent / "govoplan-campaign" / "src",
|
||||
root.parent / "govoplan-dashboard" / "src",
|
||||
root.parent / "govoplan-files" / "src",
|
||||
root.parent / "govoplan-identity" / "src",
|
||||
root.parent / "govoplan-mail" / "src",
|
||||
root.parent / "govoplan-ops" / "src",
|
||||
root.parent / "govoplan-organizations" / "src",
|
||||
root.parent / "govoplan-policy" / "src",
|
||||
root.parent / "govoplan-tenancy" / "src",
|
||||
]
|
||||
deprecated_tokens = {
|
||||
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
||||
}
|
||||
|
||||
hits: list[str] = []
|
||||
for scan_root in scan_roots:
|
||||
if not scan_root.exists():
|
||||
continue
|
||||
for path in scan_root.rglob("*.py"):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
for token, replacement in deprecated_tokens.items():
|
||||
if token in text:
|
||||
hits.append(f"{path}: deprecated {token}. {replacement}")
|
||||
|
||||
if hits:
|
||||
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
||||
print("\n".join(hits), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("Deprecated framework constant scan passed.")
|
||||
PY
|
||||
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
||||
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
||||
0|false|False|no|No)
|
||||
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
||||
;;
|
||||
auto)
|
||||
if "$PYTHON" - <<'PY'
|
||||
import importlib.util
|
||||
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
||||
PY
|
||||
then
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
else
|
||||
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
||||
"$PYTHON" - <<'PY'
|
||||
import warnings
|
||||
|
||||
from starlette.exceptions import StarletteDeprecationWarning
|
||||
|
||||
warnings.simplefilter("error", StarletteDeprecationWarning)
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/dependency-hygiene-ping")
|
||||
def ping() -> dict[str, bool]:
|
||||
return {"ok": True}
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/dependency-hygiene-ping")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"ok": True}
|
||||
|
||||
print("TestClient deprecation smoke passed.")
|
||||
PY
|
||||
fi
|
||||
|
||||
echo "Dependency hygiene check passed."
|
||||
@@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="/mnt/DATA/git/govoplan-core"
|
||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
||||
NPM="$NODE/npm"
|
||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||
|
||||
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
||||
|
||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
||||
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
||||
unset npm_config_tmp NPM_CONFIG_TMP
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||
|
||||
./.venv/bin/python - <<'PY'
|
||||
import ast
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
roots = [
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/tests"),
|
||||
]
|
||||
|
||||
errors = []
|
||||
count = 0
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
count += 1
|
||||
try:
|
||||
ast.parse(path.read_text(), filename=str(path))
|
||||
except SyntaxError as exc:
|
||||
errors.append(f"{path}:{exc.lineno}:{exc.offset}: {exc.msg}")
|
||||
|
||||
if errors:
|
||||
print("\n".join(errors))
|
||||
sys.exit(1)
|
||||
|
||||
print(f"AST syntax check passed for {count} Python files")
|
||||
PY
|
||||
|
||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||
./.venv/bin/python -m unittest tests.test_module_system
|
||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||
|
||||
cd "$ROOT/webui"
|
||||
"$NPM" run test:mail-components
|
||||
"$NPM" run test:module-capabilities
|
||||
"$NPM" run test:module-permutations
|
||||
|
||||
cd /mnt/DATA/git/govoplan-mail/webui
|
||||
"$NPM" run test:mail-ui
|
||||
|
||||
cd /mnt/DATA/git/govoplan-campaign/webui
|
||||
"$NPM" run test:policy-ui
|
||||
"$NPM" run test:template-preview
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||
NODE_BIN="$(dirname "$NPM")"
|
||||
|
||||
if [[ ! -x "$PYTHON" ]]; then
|
||||
PYTHON=python
|
||||
fi
|
||||
if [[ ! -x "$NPM" ]]; then
|
||||
NPM=npm
|
||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
||||
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||
|
||||
cd "$ROOT/webui"
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
||||
@@ -1,236 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPOS = {
|
||||
"core": ROOT / "src" / "govoplan_core",
|
||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
||||
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||
}
|
||||
PREFIXES = {
|
||||
"core": "govoplan_core",
|
||||
"access": "govoplan_access",
|
||||
"admin": "govoplan_admin",
|
||||
"tenancy": "govoplan_tenancy",
|
||||
"organizations": "govoplan_organizations",
|
||||
"identity": "govoplan_identity",
|
||||
"policy": "govoplan_policy",
|
||||
"audit": "govoplan_audit",
|
||||
"dashboard": "govoplan_dashboard",
|
||||
"files": "govoplan_files",
|
||||
"mail": "govoplan_mail",
|
||||
"campaign": "govoplan_campaign",
|
||||
}
|
||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
||||
WEBUI_REPOS = {
|
||||
"core": ROOT / "webui",
|
||||
"files": ROOT.parent / "govoplan-files" / "webui",
|
||||
"mail": ROOT.parent / "govoplan-mail" / "webui",
|
||||
"campaign": ROOT.parent / "govoplan-campaign" / "webui",
|
||||
}
|
||||
WEBUI_PACKAGES = {
|
||||
"core": "@govoplan/core-webui",
|
||||
"files": "@govoplan/files-webui",
|
||||
"mail": "@govoplan/mail-webui",
|
||||
"campaign": "@govoplan/campaign-webui",
|
||||
}
|
||||
WEBUI_IMPORT_RE = re.compile(
|
||||
r"(?:from\s+|import\s*\(\s*|require\s*\(\s*|export\s+[^;]*?\s+from\s+)"
|
||||
r"[\"'](?P<package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||
r"|import\s+[\"'](?P<side_effect_package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllowlistedImport:
|
||||
owner: str
|
||||
relative_path: str
|
||||
imported_owner: str
|
||||
reason: str
|
||||
|
||||
|
||||
# Transitional exceptions. This should remain empty; add entries only with a
|
||||
# dated removal plan and a capability/API contract issue.
|
||||
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
|
||||
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Violation:
|
||||
kind: str
|
||||
owner: str
|
||||
path: Path
|
||||
lineno: int
|
||||
imported: str
|
||||
imported_owner: str
|
||||
|
||||
|
||||
def _import_owner(module_name: str) -> str | None:
|
||||
for owner, prefix in PREFIXES.items():
|
||||
if module_name == prefix or module_name.startswith(f"{prefix}."):
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _imports(path: Path) -> list[tuple[int, str]]:
|
||||
tree = ast.parse(path.read_text(), filename=str(path))
|
||||
result: list[tuple[int, str]] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
result.append((node.lineno, alias.name))
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
result.append((node.lineno, node.module))
|
||||
return result
|
||||
|
||||
|
||||
def _relative(owner: str, path: Path) -> str:
|
||||
return path.relative_to(REPOS[owner]).as_posix()
|
||||
|
||||
|
||||
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
|
||||
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
|
||||
|
||||
|
||||
def _violations() -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for owner, root in REPOS.items():
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
for lineno, imported in _imports(path):
|
||||
imported_owner = _import_owner(imported)
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner == "access" and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in PLATFORM_OWNERS and imported_owner == "access":
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
if not _allowed(owner, path, imported_owner):
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner == "access":
|
||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
||||
violations.extend(_webui_violations())
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_package_owner(package_name: str) -> str | None:
|
||||
for owner, package in WEBUI_PACKAGES.items():
|
||||
if package_name == package:
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
def _webui_source_files(root: Path) -> list[Path]:
|
||||
source_root = root / "src"
|
||||
if not source_root.exists():
|
||||
return []
|
||||
return [
|
||||
path
|
||||
for path in source_root.rglob("*")
|
||||
if path.suffix in {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
|
||||
and "node_modules" not in path.parts
|
||||
and "dist" not in path.parts
|
||||
]
|
||||
|
||||
|
||||
def _webui_dependency_fields(payload: object) -> dict[str, object]:
|
||||
if not isinstance(payload, dict):
|
||||
return {}
|
||||
result: dict[str, object] = {}
|
||||
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, dict):
|
||||
result.update(value)
|
||||
return result
|
||||
|
||||
|
||||
def _webui_package_json_violations(owner: str, root: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
package_paths = [root / "package.json"]
|
||||
if root.name == "webui":
|
||||
package_paths.append(root.parent / "package.json")
|
||||
for package_path in tuple(dict.fromkeys(package_paths)):
|
||||
if not package_path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(package_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
return [Violation("webui-package", owner, package_path, exc.lineno, "invalid package.json", owner)]
|
||||
for package_name in _webui_dependency_fields(payload):
|
||||
imported_owner = _webui_package_owner(str(package_name))
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
if owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-package", owner, package_path, 1, str(package_name), imported_owner))
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_source_violations(owner: str, root: Path) -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for path in _webui_source_files(root):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for match in WEBUI_IMPORT_RE.finditer(text):
|
||||
package_name = match.group("package") or match.group("side_effect_package")
|
||||
imported_owner = _webui_package_owner(package_name)
|
||||
if imported_owner is None or imported_owner == owner:
|
||||
continue
|
||||
lineno = text.count("\n", 0, match.start()) + 1
|
||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
||||
return violations
|
||||
|
||||
|
||||
def _webui_violations() -> list[Violation]:
|
||||
violations: list[Violation] = []
|
||||
for owner, root in WEBUI_REPOS.items():
|
||||
if not root.exists():
|
||||
continue
|
||||
violations.extend(_webui_package_json_violations(owner, root))
|
||||
violations.extend(_webui_source_violations(owner, root))
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> int:
|
||||
violations = _violations()
|
||||
if not violations:
|
||||
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
|
||||
return 0
|
||||
|
||||
print("Dependency boundary violations:")
|
||||
for item in violations:
|
||||
print(
|
||||
f"- {item.path}:{item.lineno}: {item.owner} {item.kind} imports {item.imported} "
|
||||
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
|
||||
)
|
||||
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
|
||||
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
|
||||
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
|
||||
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
|
||||
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
|
||||
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
|
||||
args = parser.parse_args()
|
||||
|
||||
private_path = args.private_key.expanduser()
|
||||
public_path = args.public_key.expanduser() if args.public_key else None
|
||||
keyring_path = args.keyring.expanduser() if args.keyring else None
|
||||
if private_path.exists() and not args.force:
|
||||
raise SystemExit(f"Private key already exists: {private_path}")
|
||||
if public_path is not None and public_path.exists() and not args.force:
|
||||
raise SystemExit(f"Public key already exists: {public_path}")
|
||||
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
private_bytes = private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
public_base64 = base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
private_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
private_path.write_bytes(private_bytes)
|
||||
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
if public_path is not None:
|
||||
public_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
public_path.write_text(public_base64 + "\n", encoding="utf-8")
|
||||
|
||||
if keyring_path is not None:
|
||||
keyring_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring = {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": args.key_id,
|
||||
"status": args.status,
|
||||
"public_key": public_base64,
|
||||
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
],
|
||||
}
|
||||
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"private_key={private_path}")
|
||||
if public_path is not None:
|
||||
print(f"public_key={public_path}")
|
||||
if keyring_path is not None:
|
||||
print(f"keyring={keyring_path}")
|
||||
print(f"key_id={args.key_id}")
|
||||
print(f"public_key_base64={public_base64}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,377 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate and sign a GovOPlaN module package release catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
|
||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
||||
|
||||
|
||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CatalogModule:
|
||||
module_id: str
|
||||
repo: str
|
||||
python_package: str
|
||||
name: str
|
||||
description: str
|
||||
tags: tuple[str, ...]
|
||||
webui_package: str | None = None
|
||||
provides_interfaces: tuple[dict[str, object], ...] = ()
|
||||
requires_interfaces: tuple[dict[str, object], ...] = ()
|
||||
|
||||
|
||||
CATALOG_MODULES = (
|
||||
CatalogModule(
|
||||
module_id="tenancy",
|
||||
repo="govoplan-tenancy",
|
||||
python_package="govoplan-tenancy",
|
||||
name="Tenancy",
|
||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="organizations",
|
||||
repo="govoplan-organizations",
|
||||
python_package="govoplan-organizations",
|
||||
name="Organizations",
|
||||
description="Organization units, functions, and account-held function assignments.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="identity",
|
||||
repo="govoplan-identity",
|
||||
python_package="govoplan-identity",
|
||||
name="Identity",
|
||||
description="Canonical identities and links between identities and platform accounts.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="access",
|
||||
repo="govoplan-access",
|
||||
python_package="govoplan-access",
|
||||
name="Access",
|
||||
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/access-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="admin",
|
||||
repo="govoplan-admin",
|
||||
python_package="govoplan-admin",
|
||||
name="Admin",
|
||||
description="System settings, governance templates, module management, and admin shell contributions.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/admin-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="policy",
|
||||
repo="govoplan-policy",
|
||||
python_package="govoplan-policy",
|
||||
name="Policy",
|
||||
description="Policy and governance capability module.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="audit",
|
||||
repo="govoplan-audit",
|
||||
python_package="govoplan-audit",
|
||||
name="Audit",
|
||||
description="Audit-log storage and audit administration routes.",
|
||||
tags=("official", "platform-module"),
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="dashboard",
|
||||
repo="govoplan-dashboard",
|
||||
python_package="govoplan-dashboard",
|
||||
name="Dashboard",
|
||||
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/dashboard-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="files",
|
||||
repo="govoplan-files",
|
||||
python_package="govoplan-files",
|
||||
name="Files",
|
||||
description="Managed file spaces and campaign attachment integration.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/files-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="mail",
|
||||
repo="govoplan-mail",
|
||||
python_package="govoplan-mail",
|
||||
name="Mail",
|
||||
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/mail-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="campaigns",
|
||||
repo="govoplan-campaign",
|
||||
python_package="govoplan-campaign",
|
||||
name="Campaigns",
|
||||
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
||||
tags=("official", "business-module"),
|
||||
webui_package="@govoplan/campaign-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="calendar",
|
||||
repo="govoplan-calendar",
|
||||
python_package="govoplan-calendar",
|
||||
name="Calendar",
|
||||
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
||||
tags=("official", "service-module"),
|
||||
webui_package="@govoplan/calendar-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="docs",
|
||||
repo="govoplan-docs",
|
||||
python_package="govoplan-docs",
|
||||
name="Docs",
|
||||
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/docs-webui",
|
||||
),
|
||||
CatalogModule(
|
||||
module_id="ops",
|
||||
repo="govoplan-ops",
|
||||
python_package="govoplan-ops",
|
||||
name="Ops",
|
||||
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||
tags=("official", "platform-module"),
|
||||
webui_package="@govoplan/ops-webui",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
||||
parser.add_argument("--channel", default="stable")
|
||||
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
||||
parser.add_argument("--expires-days", type=int, default=90)
|
||||
parser.add_argument("--catalog-output", type=Path, required=True)
|
||||
parser.add_argument("--keyring-output", type=Path)
|
||||
parser.add_argument(
|
||||
"--catalog-signing-key",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="KEY_ID=PRIVATE_KEY",
|
||||
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
||||
)
|
||||
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
||||
parser.add_argument("--repository-base", default=GITEA_BASE)
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version.removeprefix("v")
|
||||
tag = f"v{version}"
|
||||
generated_at = datetime.now(tz=UTC)
|
||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
||||
expires_at = generated_at + timedelta(days=args.expires_days)
|
||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
||||
|
||||
catalog = _catalog_payload(
|
||||
version=version,
|
||||
tag=tag,
|
||||
channel=args.channel,
|
||||
sequence=sequence,
|
||||
generated_at=generated_at,
|
||||
expires_at=expires_at,
|
||||
repository_base=args.repository_base.rstrip("/"),
|
||||
public_base_url=args.public_base_url.rstrip("/"),
|
||||
)
|
||||
if signing_keys:
|
||||
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
||||
|
||||
output = args.catalog_output.expanduser()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
if args.keyring_output is not None:
|
||||
keyring_output = args.keyring_output.expanduser()
|
||||
keyring_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
keyring_output.write_text(
|
||||
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"catalog={output}")
|
||||
if args.keyring_output is not None:
|
||||
print(f"keyring={args.keyring_output.expanduser()}")
|
||||
print(f"channel={args.channel}")
|
||||
print(f"sequence={sequence}")
|
||||
print(f"version={version}")
|
||||
return 0
|
||||
|
||||
|
||||
def _catalog_payload(
|
||||
*,
|
||||
version: str,
|
||||
tag: str,
|
||||
channel: str,
|
||||
sequence: int,
|
||||
generated_at: datetime,
|
||||
expires_at: datetime,
|
||||
repository_base: str,
|
||||
public_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
manifests = _discovered_catalog_manifests()
|
||||
modules: list[dict[str, Any]] = []
|
||||
for module in CATALOG_MODULES:
|
||||
manifest = manifests.get(module.module_id)
|
||||
module_version = manifest.version if manifest is not None else version
|
||||
module_tag = f"v{module_version.removeprefix('v')}"
|
||||
entry: dict[str, Any] = {
|
||||
"module_id": module.module_id,
|
||||
"name": module.name,
|
||||
"description": module.description,
|
||||
"version": module_version,
|
||||
"action": "install",
|
||||
"python_package": module.python_package,
|
||||
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
|
||||
"license_features": [f"module.{module.module_id}"],
|
||||
"tags": list(module.tags),
|
||||
}
|
||||
if module.webui_package:
|
||||
entry["webui_package"] = module.webui_package
|
||||
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
|
||||
manifest_interfaces = _manifest_interface_metadata(manifest)
|
||||
entry.update(manifest_interfaces)
|
||||
if module.provides_interfaces:
|
||||
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
|
||||
if module.requires_interfaces:
|
||||
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
|
||||
modules.append(entry)
|
||||
|
||||
return {
|
||||
"catalog_version": "1",
|
||||
"channel": channel,
|
||||
"sequence": sequence,
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"expires_at": _json_datetime(expires_at),
|
||||
"release": {
|
||||
"version": version,
|
||||
"tag": tag,
|
||||
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
||||
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
||||
},
|
||||
"core_release": {
|
||||
"name": "GovOPlaN Core",
|
||||
"version": version,
|
||||
"python_package": "govoplan-core",
|
||||
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
||||
"webui_package": "@govoplan/core-webui",
|
||||
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
||||
},
|
||||
"modules": modules,
|
||||
}
|
||||
|
||||
|
||||
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
|
||||
try:
|
||||
return available_module_manifests(ignore_load_errors=True)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _manifest_interface_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
|
||||
if manifest is None:
|
||||
return {}
|
||||
payload: dict[str, object] = {}
|
||||
if manifest.provides_interfaces:
|
||||
payload["provides_interfaces"] = [
|
||||
{"name": item.name, "version": item.version}
|
||||
for item in manifest.provides_interfaces
|
||||
]
|
||||
if manifest.requires_interfaces:
|
||||
requirements: list[dict[str, object]] = []
|
||||
for item in manifest.requires_interfaces:
|
||||
requirement: dict[str, object] = {
|
||||
"name": item.name,
|
||||
"optional": item.optional,
|
||||
}
|
||||
if item.version_min is not None:
|
||||
requirement["version_min"] = item.version_min
|
||||
if item.version_max_exclusive is not None:
|
||||
requirement["version_max_exclusive"] = item.version_max_exclusive
|
||||
requirements.append(requirement)
|
||||
payload["requires_interfaces"] = requirements
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
||||
key_id, separator, path_text = value.partition("=")
|
||||
if not separator or not key_id.strip() or not path_text.strip():
|
||||
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
||||
path = Path(path_text).expanduser()
|
||||
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(private_key, Ed25519PrivateKey):
|
||||
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
||||
return key_id.strip(), private_key
|
||||
|
||||
|
||||
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
|
||||
signature_payload = dict(payload)
|
||||
signature_payload.pop("signature", None)
|
||||
signature_payload.pop("signatures", None)
|
||||
signature = private_key.sign(_canonical_bytes(signature_payload))
|
||||
return {
|
||||
"algorithm": "ed25519",
|
||||
"key_id": key_id,
|
||||
"value": base64.b64encode(signature).decode("ascii"),
|
||||
}
|
||||
|
||||
|
||||
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
|
||||
return {
|
||||
"keyring_version": "1",
|
||||
"purpose": "govoplan module package catalog signatures",
|
||||
"generated_at": _json_datetime(generated_at),
|
||||
"keys": [
|
||||
{
|
||||
"key_id": key_id,
|
||||
"status": "active",
|
||||
"public_key": _public_key_base64(private_key),
|
||||
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
||||
}
|
||||
for key_id, private_key in signing_keys
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
||||
public_bytes = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
return base64.b64encode(public_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _canonical_bytes(payload: object) -> bytes:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def _json_datetime(value: datetime) -> str:
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/generate-release-lock.sh [options]
|
||||
|
||||
Generates webui/package-lock.release.json from webui/package.release.json in a
|
||||
temporary workspace. The normal development package.json and package-lock.json
|
||||
are left untouched.
|
||||
|
||||
Run this after the module git tags referenced by package.release.json exist and
|
||||
are reachable, and before tagging the core release commit that should contain
|
||||
the regenerated release lockfile.
|
||||
|
||||
Options:
|
||||
--npm <path> npm executable to use.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WEBUI="$ROOT/webui"
|
||||
NPM_BIN="${NPM:-}"
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$WEBUI/package.release.json" ]] || fail "missing $WEBUI/package.release.json"
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm executable not found: $NPM_BIN"
|
||||
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-lock.XXXXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"
|
||||
if [[ -f "$WEBUI/package-lock.release.json" ]]; then
|
||||
cp "$WEBUI/package-lock.release.json" "$TMP_DIR/package-lock.json"
|
||||
fi
|
||||
|
||||
echo "Generating release lockfile from $WEBUI/package.release.json"
|
||||
echo "Temporary workspace: $TMP_DIR"
|
||||
(
|
||||
cd "$TMP_DIR"
|
||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
||||
)
|
||||
|
||||
cp "$TMP_DIR/package-lock.json" "$WEBUI/package-lock.release.json"
|
||||
echo "Updated $WEBUI/package-lock.release.json"
|
||||
@@ -1,580 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import markdown backlog/planning files into Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||
|
||||
REPO_ROOTS = {
|
||||
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
||||
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
||||
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
||||
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
||||
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
||||
"web": pathlib.Path("/mnt/DATA/git/govoplan-web"),
|
||||
}
|
||||
|
||||
MODULE_LABELS = {
|
||||
"core": "module/core",
|
||||
"access": "module/access",
|
||||
"mail": "module/mail",
|
||||
"files": "module/files",
|
||||
"campaign": "module/campaign",
|
||||
"web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
candidates = list(build_candidates(args))
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
print_preview(candidates)
|
||||
return 0
|
||||
|
||||
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
||||
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
||||
skipped = len(candidates) - len(missing)
|
||||
|
||||
print(f"Skipped {skipped} existing issue candidate(s).")
|
||||
print(f"Missing {len(missing)} issue candidate(s).")
|
||||
print_preview(missing)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
created_by_repo: dict[str, list[str]] = {}
|
||||
for candidate in missing:
|
||||
state = states[candidate.repo_key]
|
||||
validate_labels(candidate, state)
|
||||
issue = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": [state.label_ids[label] for label in candidate.labels],
|
||||
},
|
||||
)
|
||||
number = issue.get("number") or issue.get("index")
|
||||
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
|
||||
print("Created issues:")
|
||||
for repo_key, titles in created_by_repo.items():
|
||||
print(f"{repo_key}:")
|
||||
for title in titles:
|
||||
print(f" {title}")
|
||||
if not created_by_repo:
|
||||
print(" none")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||
if args.product_backlog.exists():
|
||||
yield from parse_product_backlog(args.product_backlog)
|
||||
if args.access_plan and args.access_plan.exists():
|
||||
yield from parse_access_plan(args.access_plan)
|
||||
if args.web_todo.exists():
|
||||
yield from parse_simple_todo(args.web_todo)
|
||||
|
||||
|
||||
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
headings: list[tuple[int, str]] = []
|
||||
status = ""
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
if level == 2 and title == "Completed Work Archive":
|
||||
break
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
status = ""
|
||||
continue
|
||||
|
||||
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
||||
if status_match:
|
||||
status = status_match.group(1).strip()
|
||||
continue
|
||||
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
section = section_title(headings)
|
||||
repo_key, labels = classify_product_item(item, section)
|
||||
prefix = type_prefix(labels)
|
||||
title = format_title(prefix, item)
|
||||
source = str(path)
|
||||
body = body_for_item(
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
section=section,
|
||||
status=status,
|
||||
item=item,
|
||||
note="Imported from the consolidated GovOPlaN product backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=tuple(sorted(set(labels))),
|
||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
||||
source=source,
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
yield from access_stage_candidates(path, lines)
|
||||
yield from access_immediate_candidates(path, lines)
|
||||
yield from access_decision_candidates(path, lines)
|
||||
|
||||
|
||||
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
||||
if not match:
|
||||
index += 1
|
||||
continue
|
||||
stage = match.group(1)
|
||||
name = match.group(2).strip()
|
||||
start_line = index + 1
|
||||
index += 1
|
||||
block: list[str] = []
|
||||
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
||||
block.append(lines[index])
|
||||
index += 1
|
||||
item = f"Access extraction Stage {stage}: {name}"
|
||||
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from the GovOPlaN access extraction plan.",
|
||||
"",
|
||||
f"- Source: `{path}`",
|
||||
f"- Line: `{start_line}`",
|
||||
f"- Section: `Stage {stage}: {name}`",
|
||||
"",
|
||||
"Plan excerpt:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"### Stage {stage}: {name}",
|
||||
*block,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=start_line,
|
||||
)
|
||||
|
||||
|
||||
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Immediate Backlog":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
title_item = access_title_item(item)
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Immediate Backlog",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", title_item),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
||||
in_section = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
if line == "## Open Decisions":
|
||||
in_section = True
|
||||
continue
|
||||
if in_section and line.startswith("## "):
|
||||
break
|
||||
if not in_section:
|
||||
continue
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
||||
body = body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="Open Decisions",
|
||||
status="decision needed",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key="core",
|
||||
title=format_title("[Task]", f"Decide: {item}"),
|
||||
body=body,
|
||||
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
||||
if not item_match:
|
||||
continue
|
||||
item = collect_continued_item(lines, index, item_match.group(1))
|
||||
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
||||
yield Candidate(
|
||||
repo_key="web",
|
||||
title=format_title("[Task]", item),
|
||||
body=body_for_item(
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
section="TODO",
|
||||
status="open",
|
||||
item=item,
|
||||
note="Imported from the GovOPlaN website TODO file.",
|
||||
),
|
||||
labels=labels("web", "type/task", "area/marketing"),
|
||||
fingerprint=fingerprint,
|
||||
source=str(path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
||||
text = f"{section} {item}".lower()
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
issue_type = "type/task"
|
||||
|
||||
if "dsar search/export/delete" in text:
|
||||
return "core", labels("core", "type/feature", "area/governance")
|
||||
if "fully stream zip uploads" in text:
|
||||
return "files", labels("files", "type/task", "area/api")
|
||||
if "installer packaging approach" in text:
|
||||
return "core", labels("core", "type/task", "area/devex")
|
||||
if "profile reselection/revalidation" in text:
|
||||
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
||||
if "module boundaries" in text:
|
||||
return "core", labels("core", issue_type, "area/module-system")
|
||||
if "external storage connectors" in text:
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
elif "controlled sending" in text:
|
||||
repo_key = "campaign"
|
||||
area = "area/api"
|
||||
elif "mail profiles" in text:
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
|
||||
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
||||
repo_key = "mail"
|
||||
area = "area/api"
|
||||
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
||||
repo_key = "files"
|
||||
area = "area/api"
|
||||
if "external storage connectors" not in text and any(
|
||||
word in text
|
||||
for word in (
|
||||
"campaign",
|
||||
"recipient",
|
||||
"attachment",
|
||||
"zip",
|
||||
"evidence",
|
||||
"wizard",
|
||||
"send",
|
||||
"archive",
|
||||
"message preview",
|
||||
"address book",
|
||||
"carddav",
|
||||
"mailing lists",
|
||||
)
|
||||
):
|
||||
repo_key = "campaign"
|
||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
||||
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
||||
repo_key = "core"
|
||||
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
||||
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
||||
repo_key = "core"
|
||||
area = "area/module-system"
|
||||
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
||||
repo_key = "core"
|
||||
area = "area/release"
|
||||
if repo_key == "core" and not any(
|
||||
phrase in text
|
||||
for phrase in (
|
||||
"external storage connectors",
|
||||
"controlled sending",
|
||||
"report, evidence",
|
||||
"recipient import",
|
||||
"campaign and attachment",
|
||||
)
|
||||
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
||||
repo_key = "core"
|
||||
area = "area/devex"
|
||||
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
||||
area = "area/docs"
|
||||
if "ui lists often start at index 2" in text:
|
||||
issue_type = "type/bug"
|
||||
area = "area/webui"
|
||||
repo_key = "core"
|
||||
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
||||
issue_type = "type/feature"
|
||||
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
||||
issue_type = "type/debt"
|
||||
|
||||
return repo_key, labels(repo_key, issue_type, area)
|
||||
|
||||
|
||||
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
||||
parts = [initial.strip()]
|
||||
next_index = index + 1
|
||||
while next_index < len(lines):
|
||||
line = lines[next_index]
|
||||
if not line.startswith((" ", "\t")):
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
break
|
||||
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
||||
break
|
||||
parts.append(stripped)
|
||||
next_index += 1
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
||||
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
||||
base.update(extra)
|
||||
if "status/needs-info" in base:
|
||||
base.discard("status/triage")
|
||||
return tuple(sorted(base))
|
||||
|
||||
|
||||
def type_prefix(label_set: Iterable[str]) -> str:
|
||||
labels_set = set(label_set)
|
||||
if "type/bug" in labels_set:
|
||||
return "[Bug]"
|
||||
if "type/feature" in labels_set:
|
||||
return "[Feature]"
|
||||
if "type/debt" in labels_set:
|
||||
return "[Debt]"
|
||||
if "type/docs" in labels_set:
|
||||
return "[Docs]"
|
||||
return "[Task]"
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
||||
title = f"{prefix} {clean}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def access_title_item(item: str) -> str:
|
||||
lowered = item.lower()
|
||||
if "compatibility wrappers" in lowered:
|
||||
return "Wire core auth compatibility through access capabilities"
|
||||
return item
|
||||
|
||||
|
||||
def body_for_item(
|
||||
*,
|
||||
fingerprint: str,
|
||||
source: str,
|
||||
line: int,
|
||||
section: str,
|
||||
status: str,
|
||||
item: str,
|
||||
note: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
note,
|
||||
"",
|
||||
f"- Source: `{source}`",
|
||||
f"- Line: `{line}`",
|
||||
f"- Section: `{section}`",
|
||||
]
|
||||
if status:
|
||||
lines.append(f"- Source status: `{status}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"Imported backlog item:",
|
||||
"",
|
||||
"```markdown",
|
||||
f"- [ ] {item}",
|
||||
"```",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([source, str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
||||
grouped: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
||||
return grouped
|
||||
|
||||
|
||||
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in repo_keys:
|
||||
root = repo_root(REPO_ROOTS[repo_key])
|
||||
target = infer_target(root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
if issue.get("state") == "closed" and "Moved to `" in body:
|
||||
continue
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
||||
return states
|
||||
|
||||
|
||||
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
||||
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
||||
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
||||
for repo_key, grouped in group_candidates(candidates).items():
|
||||
print(f"{repo_key}:")
|
||||
for candidate in grouped[:limit]:
|
||||
print(f" {candidate.title}")
|
||||
if len(grouped) > limit:
|
||||
print(f" ... {len(grouped) - limit} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post a standardized Codex state update comment to a Gitea issue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
STATUS_LABELS = {
|
||||
"started": "status/in-progress",
|
||||
"progress": "status/in-progress",
|
||||
"blocked": "status/blocked",
|
||||
"needs-info": "status/needs-info",
|
||||
"ready": "status/ready",
|
||||
"done": "",
|
||||
"note": "",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
||||
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
||||
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
||||
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
||||
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
||||
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
||||
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
||||
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
||||
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
body = build_body(args)
|
||||
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Issue: #{args.issue}")
|
||||
print(body)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
||||
body={"body": body},
|
||||
)
|
||||
if args.close:
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print("Posted Codex note.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_body(args: argparse.Namespace) -> str:
|
||||
lines = [f"## Codex State: {args.status}", ""]
|
||||
append_section(lines, "Summary", args.summary)
|
||||
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
||||
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
||||
append_section(lines, "Next / Blocked", args.next)
|
||||
if STATUS_LABELS[args.status]:
|
||||
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
||||
if args.body_file:
|
||||
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
||||
if not items:
|
||||
return
|
||||
lines.extend([f"### {title}", ""])
|
||||
lines.extend(f"- {item}" for item in items)
|
||||
lines.append("")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,597 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Iterable
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
EXCLUDED_FILE_PATTERNS = (
|
||||
"/.git/",
|
||||
"/.venv/",
|
||||
"/venv/",
|
||||
"/site-packages/",
|
||||
"/node_modules/",
|
||||
"/__pycache__/",
|
||||
"/dist/",
|
||||
"/build/",
|
||||
"/.cache/",
|
||||
"/.module-test-build/",
|
||||
)
|
||||
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
||||
ACTION_RE = re.compile(
|
||||
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
||||
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
||||
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SKIP_SECTION_RE = re.compile(
|
||||
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
||||
r"available now|done|archive)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ACTIVE_SECTION_RE = re.compile(
|
||||
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
||||
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
GENERIC_LABELS = [
|
||||
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
||||
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
||||
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
||||
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
||||
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
||||
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
||||
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
||||
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
||||
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
||||
("priority/p2", "fbca04", "Normal planned priority."),
|
||||
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
||||
]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
remote: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class SourceFile:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
source_kind: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Candidate:
|
||||
repo_key: str
|
||||
title: str
|
||||
body: str
|
||||
labels: tuple[str, ...]
|
||||
priority: str
|
||||
milestone: str
|
||||
fingerprint: str
|
||||
source: str
|
||||
line: int
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
candidates = list(build_candidates(sources))
|
||||
candidates = dedupe_candidates(candidates)
|
||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
||||
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Backlog-like source files: {len(sources)}")
|
||||
print(f"Issue candidates: {len(candidates)}")
|
||||
for repo_key, count in grouped_counts(candidates).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
print_preview(candidates)
|
||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
||||
return 0
|
||||
|
||||
missing_by_repo: dict[str, list[Candidate]] = {}
|
||||
skipped = 0
|
||||
states: dict[str, RepoState] = {}
|
||||
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
||||
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
||||
for candidate in candidates:
|
||||
state = states[candidate.repo_key]
|
||||
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
||||
skipped += 1
|
||||
continue
|
||||
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
|
||||
missing_total = sum(len(items) for items in missing_by_repo.values())
|
||||
print(f"Skipped existing: {skipped}")
|
||||
print(f"Missing candidates: {missing_total}")
|
||||
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
||||
return 0
|
||||
|
||||
for repo_key, repo_candidates in missing_by_repo.items():
|
||||
state = states[repo_key]
|
||||
for candidate in repo_candidates:
|
||||
label_ids = [state.label_ids[label] for label in candidate.labels]
|
||||
if candidate.priority not in candidate.labels:
|
||||
label_ids.append(state.label_ids[candidate.priority])
|
||||
milestone_id = ensure_milestone(state, candidate.milestone)
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
||||
body={
|
||||
"title": candidate.title,
|
||||
"body": candidate.body,
|
||||
"labels": label_ids,
|
||||
"milestone": milestone_id,
|
||||
},
|
||||
)
|
||||
number = created.get("number") or created.get("index")
|
||||
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
||||
state.fingerprints.add(candidate.fingerprint)
|
||||
state.titles.add(normalize_title(candidate.title))
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RepoState:
|
||||
target: Any
|
||||
client: GiteaClient
|
||||
label_ids: dict[str, int]
|
||||
milestone_ids: dict[str, int]
|
||||
fingerprints: set[str]
|
||||
titles: set[str]
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
remote = result.stdout.strip()
|
||||
if "git.add-ideas.de" not in remote:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
||||
sources: list[SourceFile] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
||||
continue
|
||||
path_text = path.as_posix()
|
||||
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
||||
continue
|
||||
if is_excluded_repo_file(path):
|
||||
continue
|
||||
if is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
||||
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
||||
|
||||
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.path.resolve())] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
||||
|
||||
|
||||
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
||||
text = path.as_posix()
|
||||
name = path.name.lower()
|
||||
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
|
||||
return True
|
||||
if "testing_plan" in name or "test_plan" in name:
|
||||
return True
|
||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||
return True
|
||||
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||
return True
|
||||
if text.endswith("/govoplan-web/TODO.md"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_backlog(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
if b"\x00" in chunk:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
||||
for source in sources:
|
||||
suffix = source.path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
yield from parse_csv_source(source)
|
||||
else:
|
||||
yield from parse_text_source(source)
|
||||
|
||||
|
||||
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for index, row in enumerate(reader, start=2):
|
||||
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
||||
if not story:
|
||||
continue
|
||||
identifier = (row.get("id") or "").strip()
|
||||
priority = normalize_priority(row.get("priority") or "")
|
||||
milestone = (row.get("epic") or "Backlog").strip()
|
||||
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
||||
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a CSV backlog file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{index}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Milestone: `{milestone}`",
|
||||
"",
|
||||
"CSV row:",
|
||||
"",
|
||||
"```text",
|
||||
", ".join(f"{key}={value}" for key, value in row.items()),
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=index,
|
||||
)
|
||||
|
||||
|
||||
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
||||
try:
|
||||
lines = source.path.read_text(encoding="utf-8").splitlines()
|
||||
except UnicodeDecodeError:
|
||||
lines = source.path.read_text(encoding="latin-1").splitlines()
|
||||
|
||||
headings: list[tuple[int, str]] = []
|
||||
in_tasks = False
|
||||
for index, line in enumerate(lines):
|
||||
line_number = index + 1
|
||||
heading = parse_heading(line)
|
||||
if heading:
|
||||
level, title = heading
|
||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
||||
headings.append((level, title))
|
||||
in_tasks = False
|
||||
continue
|
||||
|
||||
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
||||
in_tasks = True
|
||||
continue
|
||||
|
||||
item = parse_open_item(line)
|
||||
if item is None and is_action_context(source.path, headings, in_tasks):
|
||||
item = parse_plain_action_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
if not item or should_skip_item(item, headings):
|
||||
continue
|
||||
|
||||
section = section_title(headings)
|
||||
priority = priority_from_context(section, item)
|
||||
milestone = milestone_from_context(source.path, section)
|
||||
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
||||
title = format_title(prefix, item)
|
||||
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from a backlog-like text file.",
|
||||
"",
|
||||
f"- Source: `{source.path}`",
|
||||
f"- Line: `{line_number}`",
|
||||
f"- Source kind: `{source.source_kind}`",
|
||||
f"- Section: `{section or 'none'}`",
|
||||
"",
|
||||
"Imported item:",
|
||||
"",
|
||||
"```text",
|
||||
item,
|
||||
"```",
|
||||
]
|
||||
)
|
||||
yield Candidate(
|
||||
repo_key=source.repo_key,
|
||||
title=title,
|
||||
body=body,
|
||||
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
||||
priority=priority,
|
||||
milestone=milestone,
|
||||
fingerprint=fingerprint,
|
||||
source=str(source.path),
|
||||
line=line_number,
|
||||
)
|
||||
|
||||
|
||||
def parse_heading(line: str) -> tuple[int, str] | None:
|
||||
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
||||
if markdown:
|
||||
return len(markdown.group(1)), clean_text(markdown.group(2))
|
||||
underlined = re.match(r"^(.+?)\s*$", line)
|
||||
if underlined and line.strip() and len(line.strip()) < 120:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_open_item(line: str) -> str | None:
|
||||
patterns = [
|
||||
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
||||
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
return clean_text(match.group("text"))
|
||||
return None
|
||||
|
||||
|
||||
def parse_plain_action_item(line: str) -> str | None:
|
||||
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
||||
if match and leading_width(match.group("indent")) == 0:
|
||||
text = clean_text(match.group("text"))
|
||||
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
||||
return None
|
||||
if ACTION_RE.match(text) or is_short_noun_task(text):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
||||
if in_tasks:
|
||||
return True
|
||||
context = section_title(headings)
|
||||
if not context:
|
||||
return False
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return False
|
||||
if ACTIVE_SECTION_RE.search(context):
|
||||
return True
|
||||
return path.name.lower() in {"todo.txt", "todo.md"}
|
||||
|
||||
|
||||
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
||||
context = section_title(headings)
|
||||
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
||||
return True
|
||||
if item.endswith((",", ";")):
|
||||
return True
|
||||
if SKIP_SECTION_RE.search(context):
|
||||
return True
|
||||
if len(item) < 4:
|
||||
return True
|
||||
if item.endswith(":") and len(item.split()) <= 6:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_short_noun_task(text: str) -> bool:
|
||||
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
text = text.strip()
|
||||
text = re.sub(r"^☐\s*", "", text)
|
||||
text = re.sub(r"^✅\s*", "", text)
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip(" -")
|
||||
|
||||
|
||||
def leading_width(indent: str) -> int:
|
||||
return len(indent.replace("\t", " "))
|
||||
|
||||
|
||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
||||
return " > ".join(title for _, title in headings)
|
||||
|
||||
|
||||
def priority_from_context(section: str, item: str) -> str:
|
||||
text = f"{section} {item}".lower()
|
||||
if "p0" in text or "critical path" in text:
|
||||
return "priority/p0"
|
||||
if "p1" in text or "immediate" in text or "next sprint" in text:
|
||||
return "priority/p1"
|
||||
if "p2" in text:
|
||||
return "priority/p2"
|
||||
if "p3" in text or "future" in text or "later" in text:
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def normalize_priority(value: str) -> str:
|
||||
value = value.strip().lower()
|
||||
if value == "p0":
|
||||
return "priority/p0"
|
||||
if value == "p1":
|
||||
return "priority/p1"
|
||||
if value == "p2":
|
||||
return "priority/p2"
|
||||
if value == "p3":
|
||||
return "priority/p3"
|
||||
return "priority/p2"
|
||||
|
||||
|
||||
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
||||
if section:
|
||||
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
||||
if parts:
|
||||
return parts[-1][:120]
|
||||
return path.stem.replace("_", " ").replace("-", " ").title()
|
||||
|
||||
|
||||
def format_title(prefix: str, item: str) -> str:
|
||||
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
||||
if len(title) <= 180:
|
||||
return title
|
||||
return f"{title[:177].rstrip()}..."
|
||||
|
||||
|
||||
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
||||
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
||||
seen: set[tuple[str, str]] = set()
|
||||
unique: list[Candidate] = []
|
||||
for candidate in candidates:
|
||||
key = (candidate.repo_key, normalize_title(candidate.title))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for candidate in candidates:
|
||||
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
|
||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||
if apply:
|
||||
for name, color, description in GENERIC_LABELS:
|
||||
if name in label_ids:
|
||||
continue
|
||||
created = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/labels"),
|
||||
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
||||
)
|
||||
label_ids[str(created["name"])] = int(created["id"])
|
||||
print(f"created label {target.owner}/{target.repo}:{name}")
|
||||
|
||||
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
||||
if missing and apply:
|
||||
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
||||
|
||||
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
||||
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
||||
fingerprints: set[str] = set()
|
||||
titles: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
||||
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
||||
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
||||
|
||||
|
||||
def ensure_milestone(state: RepoState, title: str) -> int:
|
||||
if title in state.milestone_ids:
|
||||
return state.milestone_ids[title]
|
||||
created = state.client.request_json(
|
||||
"POST",
|
||||
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
||||
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
||||
)
|
||||
state.milestone_ids[title] = int(created["id"])
|
||||
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
||||
return state.milestone_ids[title]
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
||||
|
||||
|
||||
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
||||
by_repo: dict[str, list[Candidate]] = {}
|
||||
for candidate in candidates:
|
||||
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
||||
for repo_key, items in sorted(by_repo.items()):
|
||||
print(f"{repo_key}:")
|
||||
for item in items[:limit_per_repo]:
|
||||
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
||||
if len(items) > limit_per_repo:
|
||||
print(f" ... {len(items) - limit_per_repo} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install Gitea issue workflow files into one or more repositories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from gitea_common import GiteaError, repo_root
|
||||
|
||||
|
||||
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
WORKFLOW_FILES = (
|
||||
".gitea/PULL_REQUEST_TEMPLATE.md",
|
||||
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
||||
".gitea/ISSUE_TEMPLATE/config.yaml",
|
||||
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
||||
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
||||
".gitea/ISSUE_TEMPLATE/task.md",
|
||||
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
||||
)
|
||||
MODULE_LABEL_BY_REPO = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
||||
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
||||
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
||||
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
||||
unique_roots = list(dict.fromkeys(target_roots))
|
||||
if args.module_label and len(unique_roots) != 1:
|
||||
raise GiteaError("--module-label can only be used with a single target")
|
||||
|
||||
rel_paths = list(WORKFLOW_FILES)
|
||||
if args.include_labels_file:
|
||||
rel_paths.append("docs/gitea-labels.json")
|
||||
|
||||
for target_root in unique_roots:
|
||||
module_label = args.module_label or infer_module_label(target_root)
|
||||
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
||||
|
||||
if not args.apply:
|
||||
print("Dry run only. Re-run with --apply to write files.")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def infer_module_label(target_root: pathlib.Path) -> str:
|
||||
repo_name = target_root.name
|
||||
try:
|
||||
repo_name = target_root.resolve().name
|
||||
except OSError:
|
||||
pass
|
||||
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
||||
if label:
|
||||
return label
|
||||
if repo_name.startswith("govoplan-"):
|
||||
suffix = repo_name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/<name>.")
|
||||
|
||||
|
||||
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
||||
print(f"Target: {target_root} ({module_label})")
|
||||
for rel_path in rel_paths:
|
||||
source = SOURCE_ROOT / rel_path
|
||||
target = target_root / rel_path
|
||||
if not source.exists():
|
||||
raise GiteaError(f"missing workflow source file: {source}")
|
||||
|
||||
content = source.read_text(encoding="utf-8")
|
||||
content = transform_content(rel_path, content, module_label)
|
||||
|
||||
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
||||
if existing == content:
|
||||
print(f"unchanged {rel_path}")
|
||||
continue
|
||||
|
||||
action = "create" if existing is None else "update"
|
||||
print(f"{action if apply else 'would ' + action} {rel_path}")
|
||||
if apply:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
||||
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
||||
return content.replace(" - module/core", f" - {module_label}")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Move issue labels from repository-local labels to organization labels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||
parser.add_argument(
|
||||
"--issue-mode",
|
||||
choices=("replace", "delete-add"),
|
||||
default="replace",
|
||||
help="replace labels in one request, or delete local ids before adding org ids",
|
||||
)
|
||||
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
owner = args.org or target.owner
|
||||
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||
if client.token is None:
|
||||
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||
|
||||
repo_filter = re.compile(args.repo_regex)
|
||||
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||
if not org_labels:
|
||||
raise GiteaError(f"{owner} has no organization labels")
|
||||
|
||||
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||
totals = Totals()
|
||||
print(f"Organization: {owner}")
|
||||
print(f"Repositories: {len(repos)}")
|
||||
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||
|
||||
for index, repo in enumerate(repos, start=1):
|
||||
repo_result = migrate_repository(
|
||||
client,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
org_labels=org_labels,
|
||||
issue_mode=args.issue_mode,
|
||||
apply=args.apply,
|
||||
delete_repo_labels=args.delete_repo_labels,
|
||||
)
|
||||
totals.add(repo_result)
|
||||
if repo_result.has_work:
|
||||
print(
|
||||
f"[{index}/{len(repos)}] {repo}: "
|
||||
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||
f"errors={len(repo_result.errors)}"
|
||||
)
|
||||
for error in repo_result.errors:
|
||||
print(f" error: {error}")
|
||||
else:
|
||||
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||
|
||||
print("Summary:")
|
||||
print(f" repositories processed: {totals.repositories}")
|
||||
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||
print(f" errors: {totals.errors}")
|
||||
if totals.errors:
|
||||
return 1
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class RepoResult:
|
||||
def __init__(self) -> None:
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors: list[str] = []
|
||||
self.matched_local_labels = 0
|
||||
|
||||
@property
|
||||
def has_work(self) -> bool:
|
||||
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||
|
||||
|
||||
class Totals:
|
||||
def __init__(self) -> None:
|
||||
self.repositories = 0
|
||||
self.repositories_with_work = 0
|
||||
self.issue_label_migrations = 0
|
||||
self.repo_label_deletions = 0
|
||||
self.errors = 0
|
||||
|
||||
def add(self, result: RepoResult) -> None:
|
||||
self.repositories += 1
|
||||
if result.has_work:
|
||||
self.repositories_with_work += 1
|
||||
self.issue_label_migrations += result.issue_label_migrations
|
||||
self.repo_label_deletions += result.repo_label_deletions
|
||||
self.errors += len(result.errors)
|
||||
|
||||
|
||||
def migrate_repository(
|
||||
client: GiteaClient,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
org_labels: dict[str, dict[str, Any]],
|
||||
issue_mode: str,
|
||||
apply: bool,
|
||||
delete_repo_labels: bool,
|
||||
) -> RepoResult:
|
||||
result = RepoResult()
|
||||
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||
local_labels = {
|
||||
str(label.get("name") or ""): label
|
||||
for label in repo_labels
|
||||
if str(label.get("name") or "") in org_labels
|
||||
}
|
||||
result.matched_local_labels = len(local_labels)
|
||||
if not local_labels:
|
||||
return result
|
||||
|
||||
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||
|
||||
for issue_type in ("issues", "pulls"):
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "type": issue_type},
|
||||
limit=100,
|
||||
)
|
||||
for issue in issues:
|
||||
issue_number = int(issue.get("number") or issue.get("index"))
|
||||
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||
final_label_ids = list(issue_label_ids)
|
||||
changed = False
|
||||
migrated_count = 0
|
||||
local_ids_to_remove: list[int] = []
|
||||
org_ids_to_add: list[int] = []
|
||||
for label in issue.get("labels") or []:
|
||||
local_id = _label_id(label)
|
||||
if local_id is None or local_id not in local_by_id:
|
||||
continue
|
||||
name = str(label.get("name") or "")
|
||||
org_id = org_by_name.get(name)
|
||||
if org_id is None:
|
||||
continue
|
||||
local_ids_to_remove.append(local_id)
|
||||
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||
org_ids_to_add.append(org_id)
|
||||
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||
if org_id not in final_label_ids:
|
||||
final_label_ids.append(org_id)
|
||||
changed = True
|
||||
migrated_count += 1
|
||||
if changed:
|
||||
if apply:
|
||||
if issue_mode == "delete-add":
|
||||
for local_id in local_ids_to_remove:
|
||||
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||
if org_ids_to_add:
|
||||
client.request_json(
|
||||
"POST",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": org_ids_to_add},
|
||||
)
|
||||
else:
|
||||
client.request_json(
|
||||
"PUT",
|
||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||
body={"labels": final_label_ids},
|
||||
)
|
||||
result.issue_label_migrations += migrated_count
|
||||
|
||||
if delete_repo_labels:
|
||||
for name, label in sorted(local_labels.items()):
|
||||
label_id = _label_id(label)
|
||||
if label_id is None:
|
||||
result.errors.append(f"{name} has no label id")
|
||||
continue
|
||||
if apply:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
result.errors.append(f"delete repository label {name}: {exc}")
|
||||
continue
|
||||
result.repo_label_deletions += 1
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||
if explicit:
|
||||
return sorted(set(explicit))
|
||||
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||
return [name for name in names if repo_filter.search(name)]
|
||||
|
||||
|
||||
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||
|
||||
|
||||
def _label_id(label: dict[str, Any]) -> int | None:
|
||||
value = label.get("id") or label.get("index")
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
|
||||
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||
try:
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize issue labels to a Gitea repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import (
|
||||
GiteaClient,
|
||||
GiteaError,
|
||||
infer_target,
|
||||
load_dotenv,
|
||||
load_json,
|
||||
org_path,
|
||||
repo_path,
|
||||
repo_root,
|
||||
require_token,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument(
|
||||
"--scope",
|
||||
choices=("repository", "organization"),
|
||||
default="repository",
|
||||
help="sync repository labels or organization labels",
|
||||
)
|
||||
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
labels = _load_labels(args.labels_file)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
org_name = args.org or target.owner
|
||||
|
||||
if args.scope == "organization":
|
||||
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Labels file: {args.labels_file}")
|
||||
|
||||
if not args.apply and not token:
|
||||
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
||||
for label in labels:
|
||||
print(f"would ensure {label['name']} ({label['color']})")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
if args.scope == "organization":
|
||||
existing = _org_labels_by_name(client, org_name)
|
||||
else:
|
||||
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||
creates: list[dict[str, Any]] = []
|
||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
|
||||
for label in labels:
|
||||
current = existing.get(label["name"])
|
||||
if current is None:
|
||||
creates.append(label)
|
||||
continue
|
||||
|
||||
patch = _diff_label(current, label)
|
||||
if patch:
|
||||
updates.append((current, patch))
|
||||
|
||||
if not creates and not updates:
|
||||
print("No label changes needed.")
|
||||
return 0
|
||||
|
||||
for label in creates:
|
||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||
if args.apply:
|
||||
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||
|
||||
for current, patch in updates:
|
||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||
if args.apply:
|
||||
label_id = current.get("id") or current.get("index")
|
||||
if label_id is None:
|
||||
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||
client.request_json(
|
||||
"PATCH",
|
||||
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||
body=patch,
|
||||
)
|
||||
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
||||
payload = load_json(path)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"{path} must contain a JSON list")
|
||||
|
||||
labels: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(payload, start=1):
|
||||
if not isinstance(item, dict):
|
||||
raise GiteaError(f"label #{index} must be an object")
|
||||
label = {
|
||||
"name": str(item.get("name", "")).strip(),
|
||||
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
||||
"description": str(item.get("description", "")).strip(),
|
||||
"exclusive": bool(item.get("exclusive", False)),
|
||||
}
|
||||
if not label["name"]:
|
||||
raise GiteaError(f"label #{index} is missing name")
|
||||
if label["name"] in seen:
|
||||
raise GiteaError(f"duplicate label name: {label['name']}")
|
||||
if not _is_hex_color(label["color"]):
|
||||
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
||||
seen.add(label["name"])
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||
labels = client.paginate(org_path(owner, "/labels"))
|
||||
return {str(label.get("name")): label for label in labels}
|
||||
|
||||
|
||||
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, "/labels")
|
||||
return repo_path(owner, repo, "/labels")
|
||||
|
||||
|
||||
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||
if scope == "organization":
|
||||
return org_path(org, f"/labels/{label_id}")
|
||||
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||
|
||||
|
||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||
patch: dict[str, Any] = {}
|
||||
if _normalize_color(current.get("color")) != desired["color"]:
|
||||
patch["color"] = desired["color"]
|
||||
if str(current.get("description") or "").strip() != desired["description"]:
|
||||
patch["description"] = desired["description"]
|
||||
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
||||
patch["exclusive"] = desired["exclusive"]
|
||||
return patch
|
||||
|
||||
|
||||
def _normalize_color(value: Any) -> str:
|
||||
return str(value or "").strip().lstrip("#").lower()
|
||||
|
||||
|
||||
def _is_hex_color(value: str) -> bool:
|
||||
if len(value) != 6:
|
||||
return False
|
||||
return all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,637 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror project documentation files into Gitea repository wikis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
||||
|
||||
|
||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
||||
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
||||
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
||||
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
".gitea",
|
||||
".venv",
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"build",
|
||||
".cache",
|
||||
".module-test-build",
|
||||
}
|
||||
EXCLUDED_NAMES = {
|
||||
"package-lock.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"uv.lock",
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoInfo:
|
||||
root: pathlib.Path
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class WikiSource:
|
||||
repo_key: str
|
||||
path: pathlib.Path
|
||||
origin: str
|
||||
page_title: str
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
||||
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
||||
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
||||
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
load_dotenv(args.env_file)
|
||||
repos = discover_hosted_repos(args.git_root)
|
||||
if args.repo:
|
||||
requested = set(args.repo)
|
||||
repos = {key: repo for key, repo in repos.items() if key in requested}
|
||||
missing = requested - set(repos)
|
||||
if missing:
|
||||
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
||||
|
||||
sources = discover_sources(repos, args.products_root)
|
||||
if args.page:
|
||||
requested_pages = set(args.page)
|
||||
sources = [source for source in sources if source.page_title in requested_pages]
|
||||
missing_pages = requested_pages - {source.page_title for source in sources}
|
||||
if missing_pages:
|
||||
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
||||
print(f"Hosted repositories: {len(repos)}")
|
||||
print(f"Wiki source files: {len(sources)}")
|
||||
for repo_key, count in grouped_counts(sources).items():
|
||||
print(f" {repo_key}: {count}")
|
||||
|
||||
if not args.apply:
|
||||
preview_sources(sources)
|
||||
print("Dry run only. Re-run with --apply to write wiki pages.")
|
||||
return 0
|
||||
|
||||
token = require_token() if args.transport == "api" else ""
|
||||
failures = 0
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
repo = repos[repo_key]
|
||||
try:
|
||||
if args.transport == "git":
|
||||
sync_repo_wiki_git(
|
||||
repo,
|
||||
repo_sources,
|
||||
cache_dir=args.wiki_cache_dir,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
commit_message=args.commit_message,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
else:
|
||||
sync_repo_wiki(
|
||||
repo,
|
||||
repo_sources,
|
||||
token,
|
||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||
write_index=not bool(args.page),
|
||||
prune_managed=args.prune_managed and not bool(args.page),
|
||||
)
|
||||
except GiteaError as exc:
|
||||
failures += 1
|
||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||
return 1 if failures else 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
||||
repos: dict[str, RepoInfo] = {}
|
||||
for gitdir in sorted(git_root.glob("*/.git")):
|
||||
root = gitdir.parent
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
if "git.add-ideas.de" not in result.stdout:
|
||||
continue
|
||||
target = infer_target(root)
|
||||
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
||||
return repos
|
||||
|
||||
|
||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
||||
sources: list[WikiSource] = []
|
||||
for repo in repos.values():
|
||||
for path in repo.root.rglob("*"):
|
||||
if is_repo_doc(repo.root, path):
|
||||
rel = path.relative_to(repo.root)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo.key,
|
||||
path=path,
|
||||
origin="repository",
|
||||
page_title=title_for_path("Repo", rel),
|
||||
)
|
||||
)
|
||||
|
||||
product_map = {
|
||||
"co2api": "emission-api-lib",
|
||||
"govoplan": "govoplan-core",
|
||||
"meubility": "meubility-workbench",
|
||||
"mousehold": "mousehold",
|
||||
"prvnncr": "prvnncr-server",
|
||||
}
|
||||
if products_root.exists():
|
||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
||||
repo_key = product_map.get(product_dir.name)
|
||||
if not repo_key or repo_key not in repos:
|
||||
continue
|
||||
for path in product_dir.rglob("*"):
|
||||
if is_product_doc(path):
|
||||
rel = path.relative_to(product_dir)
|
||||
sources.append(
|
||||
WikiSource(
|
||||
repo_key=repo_key,
|
||||
path=path,
|
||||
origin=f"product:{product_dir.name}",
|
||||
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
||||
)
|
||||
)
|
||||
|
||||
unique: dict[tuple[str, str], WikiSource] = {}
|
||||
for source in sources:
|
||||
unique[(source.repo_key, source.page_title)] = source
|
||||
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
||||
|
||||
|
||||
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
parts = set(path.relative_to(repo_root_path).parts)
|
||||
if parts & EXCLUDED_PARTS:
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
rel = path.relative_to(repo_root_path)
|
||||
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
||||
return True
|
||||
if "generated" in rel.parts:
|
||||
return False
|
||||
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
||||
return True
|
||||
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_product_doc(path: pathlib.Path) -> bool:
|
||||
if not path.is_file() or not is_text_file(path):
|
||||
return False
|
||||
if path.name in EXCLUDED_NAMES:
|
||||
return False
|
||||
if SENSITIVE_NAME_RE.search(path.name):
|
||||
return False
|
||||
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
||||
return False
|
||||
if re.search(
|
||||
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
||||
path.name,
|
||||
re.IGNORECASE,
|
||||
) and path.suffix.lower() in {".md", ".txt"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_text_file(path: pathlib.Path) -> bool:
|
||||
if path.suffix.lower() not in TEXT_SUFFIXES:
|
||||
return False
|
||||
try:
|
||||
chunk = path.read_bytes()[:4096]
|
||||
except OSError:
|
||||
return False
|
||||
return b"\x00" not in chunk
|
||||
|
||||
|
||||
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
||||
stem = rel.with_suffix("").as_posix()
|
||||
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
||||
return f"{prefix}-{stem}"[:180]
|
||||
|
||||
|
||||
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
||||
grouped: dict[str, list[WikiSource]] = {}
|
||||
for source in sources:
|
||||
grouped.setdefault(source.repo_key, []).append(source)
|
||||
return dict(sorted(grouped.items()))
|
||||
|
||||
|
||||
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
||||
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
||||
|
||||
|
||||
def preview_sources(sources: list[WikiSource]) -> None:
|
||||
for repo_key, repo_sources in group_sources(sources).items():
|
||||
print(f"{repo_key}:")
|
||||
for source in repo_sources[:80]:
|
||||
print(f" {source.page_title} <- {source.path}")
|
||||
if len(repo_sources) > 80:
|
||||
print(f" ... {len(repo_sources) - 80} more")
|
||||
|
||||
|
||||
def sync_repo_wiki(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
token: str,
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
client = GiteaClient(target, token)
|
||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||
existing_page_names = {
|
||||
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
||||
for page in existing_pages
|
||||
if page.get("title")
|
||||
}
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
changed = put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
source.page_title,
|
||||
content,
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
put_wiki_page(
|
||||
client,
|
||||
target.owner,
|
||||
target.repo,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
existing_page_names,
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||
|
||||
|
||||
def sync_repo_wiki_git(
|
||||
repo: RepoInfo,
|
||||
sources: list[WikiSource],
|
||||
*,
|
||||
cache_dir: pathlib.Path,
|
||||
overwrite_unmanaged: bool,
|
||||
commit_message: str,
|
||||
write_index: bool = True,
|
||||
prune_managed: bool = False,
|
||||
) -> None:
|
||||
target = infer_target(repo.root)
|
||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||
index_lines = [
|
||||
f"# {target.repo} Project Wiki Index",
|
||||
"",
|
||||
f"{MANAGED_MARKER}index -->",
|
||||
"",
|
||||
"This page is generated from repository and product-directory project files.",
|
||||
"",
|
||||
]
|
||||
|
||||
for source in sources:
|
||||
content = render_wiki_content(source)
|
||||
status = write_wiki_page_file(
|
||||
wiki_root,
|
||||
source.page_title,
|
||||
content,
|
||||
overwrite_unmanaged=overwrite_unmanaged,
|
||||
)
|
||||
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
||||
|
||||
if write_index:
|
||||
write_wiki_page_file(
|
||||
wiki_root,
|
||||
"Codex-Project-Index",
|
||||
"\n".join(index_lines) + "\n",
|
||||
overwrite_unmanaged=True,
|
||||
)
|
||||
else:
|
||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||
if prune_managed and write_index:
|
||||
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||
if not git_has_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
run_git(wiki_root, "add", "-A")
|
||||
if not git_has_staged_changes(wiki_root):
|
||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||
return
|
||||
ensure_git_identity(wiki_root)
|
||||
run_git(wiki_root, "commit", "-m", commit_message)
|
||||
run_git(wiki_root, "push")
|
||||
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
||||
|
||||
|
||||
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
||||
remote = wiki_remote_for_repo(repo.root)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
||||
if (checkout / ".git").exists():
|
||||
run_git(checkout, "remote", "set-url", "origin", remote)
|
||||
run_git(checkout, "fetch", "origin")
|
||||
branch = current_branch(checkout)
|
||||
if branch:
|
||||
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
||||
else:
|
||||
run_git(checkout, "reset", "--hard")
|
||||
run_git(checkout, "clean", "-fd")
|
||||
return checkout
|
||||
if checkout.exists():
|
||||
shutil.rmtree(checkout)
|
||||
result = subprocess.run(
|
||||
["git", "clone", remote, str(checkout)],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return checkout
|
||||
checkout.mkdir(parents=True, exist_ok=True)
|
||||
run_git(checkout, "init")
|
||||
run_git(checkout, "remote", "add", "origin", remote)
|
||||
run_git(checkout, "checkout", "-b", "master")
|
||||
return checkout
|
||||
|
||||
|
||||
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
||||
remote = result.stdout.strip()
|
||||
if remote.endswith(".git"):
|
||||
return f"{remote[:-4]}.wiki.git"
|
||||
return f"{remote}.wiki.git"
|
||||
|
||||
|
||||
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
||||
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
||||
if path.exists():
|
||||
current = read_text(path)
|
||||
if current == content:
|
||||
return "unchanged"
|
||||
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
||||
return "skipped unmanaged"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return "updated"
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||
for path in sorted(wiki_root.glob("*.md")):
|
||||
if path.name in desired:
|
||||
continue
|
||||
current = read_text(path)
|
||||
if MANAGED_MARKER not in current:
|
||||
continue
|
||||
path.unlink()
|
||||
print(f"deleted stale managed wiki page {path.name}")
|
||||
|
||||
|
||||
def prune_managed_wiki_pages_api(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
existing_page_names: dict[str, str],
|
||||
sources: list[WikiSource],
|
||||
) -> None:
|
||||
desired = {source.page_title for source in sources}
|
||||
desired.add("Codex-Project-Index")
|
||||
for title, page_name in sorted(existing_page_names.items()):
|
||||
if title in desired:
|
||||
continue
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||
continue
|
||||
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||
|
||||
|
||||
def wiki_file_stem(title: str) -> str:
|
||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||
|
||||
|
||||
def git_has_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "status", "--porcelain"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return False
|
||||
if result.returncode == 1:
|
||||
return True
|
||||
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
||||
|
||||
|
||||
def current_branch(root: pathlib.Path) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
branch = result.stdout.strip()
|
||||
if result.returncode != 0 or branch == "HEAD":
|
||||
return ""
|
||||
return branch
|
||||
|
||||
|
||||
def ensure_git_identity(root: pathlib.Path) -> None:
|
||||
if not git_config_value(root, "user.email"):
|
||||
run_git(root, "config", "user.email", "codex@govoplan.local")
|
||||
if not git_config_value(root, "user.name"):
|
||||
run_git(root, "config", "user.name", "Codex")
|
||||
|
||||
|
||||
def git_config_value(root: pathlib.Path, key: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "config", "--get", key],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
|
||||
|
||||
def run_git(root: pathlib.Path, *args: str) -> None:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), *args],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
command = "git -C " + str(root) + " " + " ".join(args)
|
||||
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
||||
|
||||
|
||||
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
||||
except GiteaError as exc:
|
||||
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
||||
return []
|
||||
raise
|
||||
|
||||
|
||||
def render_wiki_content(source: WikiSource) -> str:
|
||||
raw = read_text(source.path)
|
||||
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||
header = [
|
||||
f"{MANAGED_MARKER}{fingerprint} -->",
|
||||
"",
|
||||
f"> Mirrored from `{source.path}`.",
|
||||
f"> Origin: `{source.origin}`.",
|
||||
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
if source.path.suffix.lower() == ".csv":
|
||||
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
||||
return "\n".join(header) + raw.rstrip() + "\n"
|
||||
|
||||
|
||||
def put_wiki_page(
|
||||
client: GiteaClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
title: str,
|
||||
content: str,
|
||||
existing_page_names: dict[str, str],
|
||||
*,
|
||||
overwrite_unmanaged: bool,
|
||||
) -> bool:
|
||||
body = {
|
||||
"title": title,
|
||||
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
||||
"message": f"Sync {title} from project files",
|
||||
}
|
||||
if title in existing_page_names:
|
||||
page_name = existing_page_names[title]
|
||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||
current_content = decode_wiki_content(current)
|
||||
if current_content == content:
|
||||
return False
|
||||
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
||||
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
||||
return False
|
||||
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
||||
return True
|
||||
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
||||
existing_page_names[title] = title
|
||||
return True
|
||||
|
||||
|
||||
def quote_wiki_page_name(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="+")
|
||||
|
||||
|
||||
def decode_wiki_content(page: dict[str, Any]) -> str:
|
||||
encoded = str(page.get("content_base64") or "")
|
||||
if not encoded:
|
||||
return ""
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return path.read_text(encoding="latin-1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,377 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preview or import inline TODO-style markers as Gitea issues."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||
|
||||
|
||||
MARKER_RE = re.compile(
|
||||
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
||||
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
||||
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
||||
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DEFAULT_EXCLUDES = (
|
||||
"!**/.git/**",
|
||||
"!**/.venv/**",
|
||||
"!**/node_modules/**",
|
||||
"!**/__pycache__/**",
|
||||
"!**/.module-test-build/**",
|
||||
"!**/dist/**",
|
||||
"!**/build/**",
|
||||
"!**/.cache/**",
|
||||
"!.gitea/**",
|
||||
"!docs/GITEA_ISSUES.md",
|
||||
"!scripts/gitea-todo-import.py",
|
||||
"!scripts/gitea-sync-labels.py",
|
||||
"!scripts/gitea-codex-note.py",
|
||||
"!scripts/gitea_common.py",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
||||
parser.add_argument(
|
||||
"--module-label",
|
||||
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
||||
)
|
||||
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
||||
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
||||
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
root = repo_root(args.root)
|
||||
load_dotenv(args.env_file or root / ".env")
|
||||
target = infer_target(root, args.remote)
|
||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||
|
||||
markers = scan_markers(root, include_linked=args.include_linked)
|
||||
if args.limit > 0:
|
||||
markers = markers[: args.limit]
|
||||
|
||||
module_label = args.module_label or infer_module_label(root)
|
||||
previews = [
|
||||
build_issue_preview(
|
||||
target.owner,
|
||||
target.repo,
|
||||
marker,
|
||||
module_label=module_label,
|
||||
infer_areas=not args.no_area_labels,
|
||||
extra_labels=args.extra_label,
|
||||
)
|
||||
for marker in markers
|
||||
]
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"Target: {target.display}")
|
||||
print(f"Scanned root: {root}")
|
||||
print(f"Found {len(previews)} importable marker(s).")
|
||||
for preview in previews:
|
||||
print(f"- {preview.title}")
|
||||
print(f" {preview.location}")
|
||||
print(f" labels: {', '.join(preview.labels)}")
|
||||
|
||||
if not args.apply:
|
||||
if not previews:
|
||||
return 0
|
||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
||||
return 0
|
||||
|
||||
if not previews:
|
||||
print("No issues to create.")
|
||||
return 0
|
||||
|
||||
client = GiteaClient(target, token)
|
||||
label_ids = load_label_ids(client, target.owner, target.repo)
|
||||
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
||||
if missing_labels:
|
||||
joined = ", ".join(missing_labels)
|
||||
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
||||
|
||||
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
||||
created = 0
|
||||
skipped = 0
|
||||
for preview in previews:
|
||||
if preview.fingerprint in existing_fingerprints:
|
||||
print(f"skip existing {preview.location}")
|
||||
skipped += 1
|
||||
continue
|
||||
issue = client.request_json(
|
||||
"POST",
|
||||
repo_path(target.owner, target.repo, "/issues"),
|
||||
body={
|
||||
"title": preview.title,
|
||||
"body": preview.body,
|
||||
"labels": [label_ids[label] for label in preview.labels],
|
||||
},
|
||||
)
|
||||
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
||||
created += 1
|
||||
|
||||
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
||||
return 0
|
||||
except GiteaError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class Marker:
|
||||
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
||||
self.path = path
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.marker = marker.upper()
|
||||
self.context = context
|
||||
self.text = text
|
||||
self.raw = raw.rstrip("\n")
|
||||
|
||||
@property
|
||||
def location(self) -> str:
|
||||
return f"{self.path}:{self.line}"
|
||||
|
||||
|
||||
class IssuePreview:
|
||||
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
||||
self.title = title
|
||||
self.body = body
|
||||
self.labels = labels
|
||||
self.fingerprint = fingerprint
|
||||
self.location = location
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"title": self.title,
|
||||
"body": self.body,
|
||||
"labels": self.labels,
|
||||
"fingerprint": self.fingerprint,
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
|
||||
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
||||
command = [
|
||||
"rg",
|
||||
"--json",
|
||||
"--hidden",
|
||||
"--line-number",
|
||||
"--column",
|
||||
"-e",
|
||||
r"\b(TODO|FIXME|XXX|HACK)\b",
|
||||
]
|
||||
for pattern in DEFAULT_EXCLUDES:
|
||||
command.extend(["--glob", pattern])
|
||||
command.append(str(root))
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode not in {0, 1}:
|
||||
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
||||
|
||||
markers: list[Marker] = []
|
||||
for line in result.stdout.splitlines():
|
||||
event = json.loads(line)
|
||||
if event.get("type") != "match":
|
||||
continue
|
||||
data = event["data"]
|
||||
raw_line = data["lines"]["text"]
|
||||
match = MARKER_RE.search(raw_line)
|
||||
if not match:
|
||||
continue
|
||||
if not _looks_like_marker(match):
|
||||
continue
|
||||
context = (match.group("context") or "").strip()
|
||||
if not include_linked and _already_linked(context, raw_line):
|
||||
continue
|
||||
markers.append(
|
||||
Marker(
|
||||
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
||||
line=int(data["line_number"]),
|
||||
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
||||
marker=match.group("marker"),
|
||||
context=context,
|
||||
text=clean_marker_text(match.group("text") or raw_line),
|
||||
raw=raw_line,
|
||||
)
|
||||
)
|
||||
return markers
|
||||
|
||||
|
||||
def build_issue_preview(
|
||||
owner: str,
|
||||
repo: str,
|
||||
marker: Marker,
|
||||
*,
|
||||
module_label: str,
|
||||
infer_areas: bool,
|
||||
extra_labels: list[str],
|
||||
) -> IssuePreview:
|
||||
fingerprint = marker_fingerprint(owner, repo, marker)
|
||||
summary = marker.text or f"review {marker.location}"
|
||||
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
||||
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
||||
body = "\n".join(
|
||||
[
|
||||
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
||||
"",
|
||||
"Imported from an inline marker.",
|
||||
"",
|
||||
f"- Repository: `{owner}/{repo}`",
|
||||
f"- Location: `{marker.location}`",
|
||||
f"- Marker: `{marker.marker}`",
|
||||
f"- Context: `{marker.context or 'none'}`",
|
||||
"",
|
||||
"Current line:",
|
||||
"",
|
||||
"```text",
|
||||
marker.raw,
|
||||
"```",
|
||||
"",
|
||||
"Migration rule:",
|
||||
"",
|
||||
"- Keep this Gitea issue as the canonical backlog item.",
|
||||
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
||||
]
|
||||
)
|
||||
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
||||
|
||||
|
||||
def clean_marker_text(text: str) -> str:
|
||||
cleaned = text.strip()
|
||||
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
||||
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
||||
return cleaned
|
||||
|
||||
|
||||
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
||||
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
||||
area = infer_area_label(marker.path) if infer_areas else ""
|
||||
if area:
|
||||
labels.append(area)
|
||||
return labels
|
||||
|
||||
|
||||
def marker_type_label(marker: str) -> str:
|
||||
if marker == "FIXME":
|
||||
return "type/bug"
|
||||
if marker in {"HACK", "XXX"}:
|
||||
return "type/debt"
|
||||
return "type/task"
|
||||
|
||||
|
||||
def infer_area_label(path: pathlib.Path) -> str:
|
||||
text = path.as_posix()
|
||||
if text.startswith("webui/"):
|
||||
return "area/webui"
|
||||
if text.startswith("alembic/"):
|
||||
return "area/migrations"
|
||||
if text.startswith("docs/"):
|
||||
return "area/docs"
|
||||
if text.startswith("scripts/"):
|
||||
return "area/devex"
|
||||
if "/rbac" in text or "permission" in text:
|
||||
return "area/rbac"
|
||||
if "/access" in text or "auth" in text or "session" in text:
|
||||
return "area/auth"
|
||||
if "tenant" in text:
|
||||
return "area/tenancy"
|
||||
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
||||
return "area/governance"
|
||||
if "module" in text:
|
||||
return "area/module-system"
|
||||
if "migration" in text:
|
||||
return "area/migrations"
|
||||
if "router" in text or "api" in text:
|
||||
return "area/api"
|
||||
if "db" in text or "model" in text or "persistence" in text:
|
||||
return "area/db"
|
||||
return ""
|
||||
|
||||
|
||||
def infer_module_label(root: pathlib.Path) -> str:
|
||||
known = {
|
||||
"govoplan-core": "module/core",
|
||||
"govoplan-access": "module/access",
|
||||
"govoplan-admin": "module/admin",
|
||||
"govoplan-tenancy": "module/tenancy",
|
||||
"govoplan-policy": "module/policy",
|
||||
"govoplan-audit": "module/audit",
|
||||
"govoplan-mail": "module/mail",
|
||||
"govoplan-files": "module/files",
|
||||
"govoplan-campaign": "module/campaign",
|
||||
"govoplan-web": "module/web",
|
||||
}
|
||||
if root.name in known:
|
||||
return known[root.name]
|
||||
if root.name.startswith("govoplan-"):
|
||||
suffix = root.name.removeprefix("govoplan-")
|
||||
if suffix:
|
||||
return f"module/{suffix}"
|
||||
return "module/core"
|
||||
|
||||
|
||||
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
||||
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def truncate_title(title: str, limit: int = 180) -> str:
|
||||
if len(title) <= limit:
|
||||
return title
|
||||
return f"{title[: limit - 1].rstrip()}..."
|
||||
|
||||
|
||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||
return label_ids_by_name(client, owner, repo)
|
||||
|
||||
|
||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||
issues = client.paginate(
|
||||
repo_path(owner, repo, "/issues"),
|
||||
query={"state": "all", "labels": "source/todo-scan"},
|
||||
)
|
||||
fingerprints: set[str] = set()
|
||||
for issue in issues:
|
||||
body = str(issue.get("body") or "")
|
||||
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
||||
fingerprints.add(match.group(1))
|
||||
return fingerprints
|
||||
|
||||
|
||||
def _already_linked(context: str, raw_line: str) -> bool:
|
||||
linked_text = f"{context} {raw_line}".lower()
|
||||
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
||||
|
||||
|
||||
def _looks_like_marker(match: re.Match[str]) -> bool:
|
||||
text = (match.group("text") or "").strip()
|
||||
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for Gitea maintenance scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
class GiteaError(RuntimeError):
|
||||
"""Raised when a Gitea API request fails."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class RepoTarget:
|
||||
base_url: str
|
||||
owner: str
|
||||
repo: str
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
||||
|
||||
@property
|
||||
def api_base(self) -> str:
|
||||
return f"{self.base_url.rstrip('/')}/api/v1"
|
||||
|
||||
|
||||
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return start.resolve()
|
||||
return pathlib.Path(result.stdout.strip()).resolve()
|
||||
|
||||
|
||||
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
||||
env_url = os.environ.get("GITEA_URL")
|
||||
env_owner = os.environ.get("GITEA_OWNER")
|
||||
env_repo = os.environ.get("GITEA_REPO")
|
||||
|
||||
remote_url = _git_remote(root, remote_name)
|
||||
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
||||
|
||||
base_url = (env_url or inferred_url).rstrip("/")
|
||||
owner = env_owner or inferred_owner
|
||||
repo = env_repo or inferred_repo
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name, value in (
|
||||
("GITEA_URL", base_url),
|
||||
("GITEA_OWNER", owner),
|
||||
("GITEA_REPO", repo),
|
||||
)
|
||||
if not value
|
||||
]
|
||||
if missing:
|
||||
joined = ", ".join(missing)
|
||||
raise GiteaError(
|
||||
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
||||
)
|
||||
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
||||
|
||||
|
||||
def quote_path(value: str) -> str:
|
||||
return urllib.parse.quote(value, safe="")
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
||||
self.target = target
|
||||
self.token = token
|
||||
|
||||
def request_json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, Any] | None = None,
|
||||
query: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{self.target.api_base}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
||||
|
||||
data = None
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "codex-gitea-maintenance",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
payload = response.read().decode("utf-8")
|
||||
if not payload:
|
||||
return None
|
||||
return json.loads(payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
message = exc.read().decode("utf-8", errors="replace")
|
||||
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
query: dict[str, Any] | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
effective_limit = min(limit, 50)
|
||||
seen_pages: set[tuple[Any, ...]] = set()
|
||||
while True:
|
||||
page_query = dict(query or {})
|
||||
page_query.update({"page": page, "limit": effective_limit})
|
||||
payload = self.request_json("GET", path, query=page_query)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
||||
if not payload:
|
||||
return items
|
||||
signature = tuple(
|
||||
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
||||
for item in payload
|
||||
if isinstance(item, dict)
|
||||
)
|
||||
if signature in seen_pages:
|
||||
return items
|
||||
seen_pages.add(signature)
|
||||
items.extend(payload)
|
||||
if len(payload) < effective_limit:
|
||||
return items
|
||||
page += 1
|
||||
|
||||
|
||||
def load_json(path: pathlib.Path) -> Any:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
||||
if path is None or not path.exists():
|
||||
return []
|
||||
|
||||
loaded: list[str] = []
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if stripped.startswith("export "):
|
||||
stripped = stripped[len("export ") :].strip()
|
||||
if "=" not in stripped:
|
||||
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = _strip_env_value(value.strip())
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||||
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
||||
if key not in os.environ:
|
||||
os.environ[key] = value
|
||||
loaded.append(key)
|
||||
return loaded
|
||||
|
||||
|
||||
def require_token() -> str:
|
||||
token = os.environ.get("GITEA_TOKEN")
|
||||
if not token:
|
||||
raise GiteaError(
|
||||
"GITEA_TOKEN is required when using --apply. "
|
||||
"Set it in the environment, the target .env, or --env-file."
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||
|
||||
|
||||
def org_path(owner: str, suffix: str) -> str:
|
||||
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||
|
||||
|
||||
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Return org labels plus optional repository labels, keyed by label name.
|
||||
|
||||
Gitea stores organization labels separately from repository labels. Issue
|
||||
APIs can use label ids from both scopes on supported instances, so issue
|
||||
creation helpers should resolve both. Repository labels intentionally win
|
||||
when a repo carries a local label with the same name.
|
||||
"""
|
||||
|
||||
labels: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
for label in client.paginate(org_path(owner, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
except GiteaError:
|
||||
# User-owned repositories or older instances may not expose org labels.
|
||||
pass
|
||||
|
||||
if repo:
|
||||
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||
name = str(label.get("name") or "")
|
||||
if name:
|
||||
labels[name] = label
|
||||
return labels
|
||||
|
||||
|
||||
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||
ids: dict[str, int] = {}
|
||||
for name, label in labels_by_name(client, owner, repo).items():
|
||||
label_id = label.get("id") or label.get("index")
|
||||
if label_id is None:
|
||||
continue
|
||||
ids[name] = int(label_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
||||
if not remote_url:
|
||||
return "", "", ""
|
||||
|
||||
parsed = urllib.parse.urlparse(remote_url)
|
||||
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
||||
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
prefix = "/".join(path_parts[:-2])
|
||||
base_path = f"/{prefix}" if prefix else ""
|
||||
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
||||
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
||||
|
||||
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
||||
if scp_like:
|
||||
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
||||
owner, repo = _owner_repo_from_parts(path_parts)
|
||||
return f"https://{scp_like.group('host')}", owner, repo
|
||||
|
||||
return "", "", ""
|
||||
|
||||
|
||||
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
||||
if len(path_parts) < 2:
|
||||
return "", ""
|
||||
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
||||
|
||||
|
||||
def _strip_git_suffix(value: str) -> str:
|
||||
return value[:-4] if value.endswith(".git") else value
|
||||
|
||||
|
||||
def _strip_env_value(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
return value[1:-1]
|
||||
return value
|
||||
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||
WEBUI_ROOT="$ROOT/webui"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||
NPM="${NPM:-$NODE_BIN/npm}"
|
||||
|
||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||
|
||||
backend_pid=""
|
||||
frontend_pid=""
|
||||
|
||||
fail() {
|
||||
printf 'launch-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
case "$DEV_DATABASE_BACKEND" in
|
||||
postgres)
|
||||
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||
;;
|
||||
sqlite)
|
||||
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
||||
;;
|
||||
*)
|
||||
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
||||
;;
|
||||
esac
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind((host, port))
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
"$PYTHON" - "$1" <<'PY'
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
deadline = time.monotonic() + 60
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if 200 <= response.status < 500:
|
||||
raise SystemExit(0)
|
||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||
kill "$frontend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||
kill "$backend_pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
|
||||
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
|
||||
fi
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||
) >"$BACKEND_LOG" 2>&1 &
|
||||
backend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||
wait_for_url "$BACKEND_URL/health" || {
|
||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||
fail "backend did not become healthy"
|
||||
}
|
||||
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
frontend_args=(run dev)
|
||||
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
|
||||
frontend_args+=(-- --force)
|
||||
fi
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \
|
||||
CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" "${frontend_args[@]}"
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||
wait_for_url "$FRONTEND_URL" || {
|
||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||
fail "frontend did not become reachable"
|
||||
}
|
||||
|
||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
GovOPlaN is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
DB: $DATABASE_URL
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Frontend reload:
|
||||
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
|
||||
Vite --force: $FRONTEND_FORCE_RELOAD
|
||||
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
|
||||
|
||||
Press Ctrl+C to stop both processes.
|
||||
EOF
|
||||
|
||||
wait
|
||||
@@ -1,241 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||
if [ ! -f "$PROFILE_ENV" ]; then
|
||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||
fi
|
||||
|
||||
WEBUI_ROOT="$ROOT/webui"
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||
NPM="${NPM:-$NODE_BIN/npm}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
|
||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
||||
|
||||
LOG_DIR="$ROOT/runtime/production-like/logs"
|
||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||
WORKER_LOG="$LOG_DIR/worker.log"
|
||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||
|
||||
backend_pid=""
|
||||
worker_pid=""
|
||||
frontend_pid=""
|
||||
|
||||
fail() {
|
||||
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$PROFILE_ENV"
|
||||
set +a
|
||||
|
||||
export APP_ENV="${APP_ENV:-staging}"
|
||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
||||
|
||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
||||
from cryptography.fernet import Fernet
|
||||
print(Fernet.generate_key().decode())
|
||||
PY
|
||||
)"
|
||||
export MASTER_KEY_B64
|
||||
fi
|
||||
|
||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||
|
||||
compose() {
|
||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
port_is_free() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
try:
|
||||
sock.bind((host, port))
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_tcp() {
|
||||
"$PYTHON" - "$1" "$2" <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
host = sys.argv[1]
|
||||
port = int(sys.argv[2])
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=2):
|
||||
raise SystemExit(0)
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
"$PYTHON" - "$1" <<'PY'
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1]
|
||||
deadline = time.monotonic() + 90
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if 200 <= response.status < 500:
|
||||
raise SystemExit(0)
|
||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||
kill "$frontend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||
kill "$backend_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
||||
kill "$worker_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
||||
compose down >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||
|
||||
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
||||
: > "$BACKEND_LOG"
|
||||
: > "$WORKER_LOG"
|
||||
: > "$FRONTEND_LOG"
|
||||
|
||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||
|
||||
printf 'Starting production-like dependencies through Docker Compose\n'
|
||||
compose up -d
|
||||
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
||||
|
||||
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||
)
|
||||
|
||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||
) >"$BACKEND_LOG" 2>&1 &
|
||||
backend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||
wait_for_url "$BACKEND_URL/health" || {
|
||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||
fail "backend did not become healthy"
|
||||
}
|
||||
|
||||
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
||||
(
|
||||
cd "$ROOT"
|
||||
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
||||
--queues "$CELERY_QUEUES" \
|
||||
--hostname "govoplan-production-like@%h" \
|
||||
--loglevel INFO
|
||||
) >"$WORKER_LOG" 2>&1 &
|
||||
worker_pid="$!"
|
||||
|
||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||
(
|
||||
cd "$WEBUI_ROOT"
|
||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
||||
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||
"$NPM" run dev -- --force
|
||||
) >"$FRONTEND_LOG" 2>&1 &
|
||||
frontend_pid="$!"
|
||||
|
||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||
wait_for_url "$FRONTEND_URL" || {
|
||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||
fail "frontend did not become reachable"
|
||||
}
|
||||
|
||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
GovOPlaN production-like development profile is running.
|
||||
Web UI: $FRONTEND_URL
|
||||
API: $BACKEND_URL/api/v1
|
||||
Health: $BACKEND_URL/health
|
||||
Ops: $BACKEND_URL/api/v1/ops/status
|
||||
DB: $DATABASE_URL
|
||||
Redis: $REDIS_URL
|
||||
Files: $FILE_STORAGE_LOCAL_ROOT
|
||||
|
||||
Logs:
|
||||
Backend: $BACKEND_LOG
|
||||
Worker: $WORKER_LOG
|
||||
WebUI: $FRONTEND_LOG
|
||||
|
||||
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
||||
Press Ctrl+C to stop API, worker, and WebUI.
|
||||
EOF
|
||||
|
||||
wait
|
||||
@@ -1,512 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
os.environ.setdefault("APP_ENV", "test")
|
||||
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
||||
os.environ.setdefault("CELERY_ENABLED", "false")
|
||||
|
||||
from govoplan_core.admin.models import SystemSettings
|
||||
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
||||
from govoplan_core.core.module_installer import (
|
||||
cancel_module_installer_request,
|
||||
claim_next_module_installer_request,
|
||||
module_installer_daemon_status,
|
||||
module_installer_lock_status,
|
||||
queue_module_installer_request,
|
||||
read_module_installer_run,
|
||||
retry_module_installer_request,
|
||||
rollback_module_install_run,
|
||||
run_module_install_plan,
|
||||
supervise_module_install_plan,
|
||||
update_module_installer_daemon_status,
|
||||
update_module_installer_request,
|
||||
)
|
||||
from govoplan_core.core.module_management import (
|
||||
ModuleInstallPlan,
|
||||
ModuleInstallPlanItem,
|
||||
saved_desired_enabled_modules,
|
||||
saved_module_install_plan,
|
||||
save_desired_enabled_modules,
|
||||
save_module_install_plan,
|
||||
)
|
||||
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.server.registry import available_module_manifests
|
||||
|
||||
|
||||
Scenario = Callable[[Path], dict[str, object]]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
||||
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
||||
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
||||
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
||||
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
||||
args = parser.parse_args()
|
||||
|
||||
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
||||
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||
selected = args.scenario or sorted(SCENARIOS)
|
||||
results: list[dict[str, object]] = []
|
||||
ok = True
|
||||
try:
|
||||
for name in selected:
|
||||
scenario_root = runtime_root / name
|
||||
scenario_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
result = SCENARIOS[name](scenario_root)
|
||||
except Exception as exc:
|
||||
ok = False
|
||||
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
||||
else:
|
||||
ok = ok and bool(result.get("ok"))
|
||||
results.append(result)
|
||||
finally:
|
||||
if not args.keep_runtime and args.runtime_root is None:
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
|
||||
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
||||
if args.format == "json":
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
for result in results:
|
||||
status = "ok" if result.get("ok") else "FAILED"
|
||||
print(f"{status}: {result['scenario']}")
|
||||
if result.get("run_id"):
|
||||
print(f" run: {result['run_id']}")
|
||||
if result.get("record_path"):
|
||||
print(f" record: {result['record_path']}")
|
||||
if result.get("error"):
|
||||
print(f" error: {result['error']}")
|
||||
print(f"runtime: {runtime_root}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def package_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "package-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_package_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
restored_plan = saved_module_install_plan(session)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"package-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def migration_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "migration-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"migration-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def restart_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "restart-failure-example")
|
||||
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-fails",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"restart-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
||||
)
|
||||
|
||||
|
||||
def health_timeout(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
plan = _saved_install_plan(database, "health-timeout-example")
|
||||
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
||||
result = supervise_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
restart_command="restart-ok",
|
||||
health_url="http://127.0.0.1:9/health",
|
||||
health_timeout_seconds=0.2,
|
||||
health_interval_seconds=0.05,
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"health-timeout",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
||||
database_url, database = _sqlite_database(root)
|
||||
available = {"retirement-example": _retirement_failure_manifest()}
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "retirement-example",
|
||||
"action": "uninstall",
|
||||
"python_package": "govoplan-retirement-example",
|
||||
"destroy_data": True,
|
||||
}])
|
||||
session.commit()
|
||||
with _patch_subprocess(_success_run):
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available,
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url=database_url,
|
||||
runtime_dir=root / "installer",
|
||||
)
|
||||
record = _run_record(root, result.run_id)
|
||||
return _scenario_result(
|
||||
"destructive-retirement-failure",
|
||||
root,
|
||||
result,
|
||||
expected_status="rolled-back",
|
||||
checks={
|
||||
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def postgres_hooks(root: Path) -> dict[str, object]:
|
||||
_, database = _sqlite_database(root)
|
||||
backup_script = (
|
||||
"import json, os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
||||
)
|
||||
check_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
||||
)
|
||||
restore_script = (
|
||||
"import os, pathlib; "
|
||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
||||
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
||||
)
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": "postgres-hook-example",
|
||||
"action": "install",
|
||||
"python_package": "govoplan-postgres-hook-example",
|
||||
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
||||
}])
|
||||
session.commit()
|
||||
result = run_module_install_plan(
|
||||
session=session,
|
||||
plan=plan,
|
||||
available=available_module_manifests(),
|
||||
current_enabled=("tenancy", "access"),
|
||||
desired_enabled=("tenancy", "access"),
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
runtime_dir=root / "installer",
|
||||
migrate_database=True,
|
||||
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
||||
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
||||
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
||||
dry_run=True,
|
||||
)
|
||||
with _patch_subprocess(_non_shell_success_run):
|
||||
rollback = rollback_module_install_run(
|
||||
run_id=result.run_id,
|
||||
runtime_dir=root / "installer",
|
||||
database_url="postgresql://db.example.invalid/govoplan",
|
||||
)
|
||||
run_dir = result.record_path.parent
|
||||
record = _run_record(root, result.run_id)
|
||||
ok = (
|
||||
result.status == "dry-run"
|
||||
and rollback.status == "rolled-back"
|
||||
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
||||
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
||||
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
||||
)
|
||||
return {
|
||||
"scenario": "postgres-hooks",
|
||||
"ok": ok,
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"rollback_status": rollback.status,
|
||||
}
|
||||
|
||||
|
||||
def queue_and_lock(root: Path) -> dict[str, object]:
|
||||
runtime_dir = root / "installer"
|
||||
status = update_module_installer_daemon_status(
|
||||
runtime_dir=runtime_dir,
|
||||
patch={"status": "polling", "current_request_id": None},
|
||||
)
|
||||
request = queue_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
requested_by="drill",
|
||||
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
||||
)
|
||||
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
||||
assert claimed is not None
|
||||
failed = update_module_installer_request(
|
||||
runtime_dir=runtime_dir,
|
||||
request_id=str(claimed["request_id"]),
|
||||
patch={"status": "failed", "error": "simulated drill failure"},
|
||||
)
|
||||
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
||||
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
||||
lock_path = runtime_dir / "install.lock"
|
||||
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
||||
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
||||
lock_path.unlink()
|
||||
return {
|
||||
"scenario": "queue-and-lock",
|
||||
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
||||
"root": str(root),
|
||||
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
||||
}
|
||||
|
||||
|
||||
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
||||
database_url = f"sqlite:///{root / 'drill.db'}"
|
||||
configure_database(database_url)
|
||||
database = get_database()
|
||||
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
||||
return database_url, database
|
||||
|
||||
|
||||
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
||||
with database.session() as session:
|
||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||
plan = save_module_install_plan(session, [{
|
||||
"module_id": module_id,
|
||||
"action": "install",
|
||||
"python_package": f"govoplan-{module_id}",
|
||||
"python_ref": f"govoplan-{module_id}==0.1.0",
|
||||
}])
|
||||
save_desired_enabled_modules(session, ("tenancy", "access"))
|
||||
session.commit()
|
||||
return plan
|
||||
|
||||
|
||||
def _retirement_failure_manifest() -> ModuleManifest:
|
||||
def destroy(_session: object, _module_id: str) -> None:
|
||||
raise RuntimeError("simulated destructive retirement failure")
|
||||
|
||||
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
||||
return MigrationRetirementPlan(
|
||||
supported=True,
|
||||
summary="Drill retirement provider supports destructive cleanup.",
|
||||
destroy_data_supported=True,
|
||||
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
||||
destroy_data_executor=destroy,
|
||||
)
|
||||
|
||||
return ModuleManifest(
|
||||
id="retirement-example",
|
||||
name="Retirement example",
|
||||
version="0.1.0",
|
||||
migration_spec=MigrationSpec(
|
||||
module_id="retirement-example",
|
||||
retirement_supported=True,
|
||||
retirement_provider=provider,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _scenario_result(
|
||||
scenario: str,
|
||||
root: Path,
|
||||
result: Any,
|
||||
*,
|
||||
expected_status: str,
|
||||
checks: dict[str, bool],
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"scenario": scenario,
|
||||
"ok": result.status == expected_status and all(checks.values()),
|
||||
"root": str(root),
|
||||
"run_id": result.run_id,
|
||||
"record_path": str(result.record_path),
|
||||
"status": result.status,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
||||
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
||||
|
||||
|
||||
def _nested(value: dict[str, object], *keys: str) -> object:
|
||||
current: object = value
|
||||
for key in keys:
|
||||
if not isinstance(current, dict):
|
||||
return None
|
||||
current = current.get(key)
|
||||
return current
|
||||
|
||||
|
||||
def _command_text(argv: object) -> str:
|
||||
if isinstance(argv, list | tuple):
|
||||
return " ".join(str(item) for item in argv)
|
||||
return str(argv)
|
||||
|
||||
|
||||
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
||||
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip freeze" in text:
|
||||
return _completed(stdout="govoplan-core==0.0.0\n")
|
||||
return _completed()
|
||||
|
||||
|
||||
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
||||
if kwargs.get("shell"):
|
||||
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "pip install" in text and "==" in text and "-r" not in text:
|
||||
return _completed(23, stderr="simulated package install failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
text = _command_text(argv)
|
||||
if "govoplan_core.commands.init_db" in text:
|
||||
return _completed(24, stderr="simulated migration failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
||||
return _completed(25, stderr="simulated restart failure")
|
||||
return _success_run(argv, *_args, **kwargs)
|
||||
|
||||
|
||||
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.subprocess.run
|
||||
installer.subprocess.run = fake_run # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.subprocess.run = original # type: ignore[assignment]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_urlopen() -> Iterable[None]:
|
||||
import govoplan_core.core.module_installer as installer
|
||||
|
||||
original = installer.urllib.request.urlopen
|
||||
|
||||
def fail(*_args: object, **_kwargs: object) -> None:
|
||||
raise URLError("simulated health timeout")
|
||||
|
||||
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
||||
|
||||
|
||||
SCENARIOS: dict[str, Scenario] = {
|
||||
"destructive-retirement-failure": destructive_retirement_failure,
|
||||
"health-timeout": health_timeout,
|
||||
"migration-failure": migration_failure,
|
||||
"package-failure": package_failure,
|
||||
"postgres-hooks": postgres_hooks,
|
||||
"queue-and-lock": queue_and_lock,
|
||||
"restart-failure": restart_failure,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,128 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
|
||||
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
||||
("core-only", "tenancy,access,admin"),
|
||||
("files-only", "tenancy,access,admin,files"),
|
||||
("mail-only", "tenancy,access,admin,mail"),
|
||||
("campaign-only", "tenancy,access,admin,campaigns"),
|
||||
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
||||
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
||||
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database-url",
|
||||
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
||||
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--module-set",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME=MODULES",
|
||||
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reset-schema",
|
||||
action="store_true",
|
||||
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
||||
)
|
||||
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
||||
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
||||
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
||||
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
||||
args = parser.parse_args()
|
||||
|
||||
database_url = (args.database_url or "").strip()
|
||||
if not database_url:
|
||||
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
||||
if not database_url.startswith("postgresql"):
|
||||
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
||||
|
||||
module_sets = _parse_module_sets(args.module_set)
|
||||
ok = True
|
||||
for name, modules in module_sets:
|
||||
print(f"\n== {name}: {modules} ==")
|
||||
if args.reset_schema:
|
||||
_reset_public_schema(database_url)
|
||||
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
||||
if not args.skip_migrate:
|
||||
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
||||
if not args.skip_smoke:
|
||||
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
||||
if not raw:
|
||||
return DEFAULT_MODULE_SETS
|
||||
parsed: list[tuple[str, str]] = []
|
||||
for item in raw:
|
||||
name, separator, modules = item.partition("=")
|
||||
if not separator or not name.strip() or not modules.strip():
|
||||
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
||||
parsed.append((name.strip(), modules.strip()))
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env["APP_ENV"] = app_env
|
||||
env["DATABASE_URL"] = database_url
|
||||
env["ENABLED_MODULES"] = modules
|
||||
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
||||
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
||||
env["CELERY_ENABLED"] = "false"
|
||||
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
||||
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
||||
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
||||
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
||||
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
||||
return env
|
||||
|
||||
|
||||
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
||||
print("+ " + " ".join(command))
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
||||
if result.returncode != 0:
|
||||
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _reset_public_schema(database_url: str) -> None:
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
||||
|
||||
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
||||
connection.exec_driver_sql("CREATE SCHEMA public")
|
||||
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
||||
finally:
|
||||
engine.dispose()
|
||||
print("Reset PostgreSQL schema: public")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||
if [ ! -f "$PROFILE_ENV" ]; then
|
||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||
fi
|
||||
|
||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||
DOCKER="${DOCKER:-docker}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/production-like-dev.sh <command> [--yes]
|
||||
|
||||
Commands:
|
||||
start Start the production-like API, worker, WebUI, PostgreSQL, and Redis profile.
|
||||
stop Stop PostgreSQL and Redis profile containers.
|
||||
status Show profile container status and validate the effective environment.
|
||||
seed Run migrations and seed development data against the profile database.
|
||||
reset --yes Stop containers, remove profile volumes, and remove runtime/production-like data.
|
||||
validate-config Validate the profile environment only.
|
||||
|
||||
The start command delegates to scripts/launch-production-like-dev.sh. Stop/reset
|
||||
operate on Docker dependencies; API/WebUI/worker processes started by the
|
||||
launcher are stopped with Ctrl+C in that launcher terminal.
|
||||
EOF
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'production-like-dev: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command="${1:-}"
|
||||
if [ -z "$command" ] || [ "$command" = "-h" ] || [ "$command" = "--help" ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
shift || true
|
||||
|
||||
yes=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--yes) yes=1 ;;
|
||||
*) fail "unknown argument: $arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$PROFILE_ENV"
|
||||
set +a
|
||||
|
||||
export APP_ENV="${APP_ENV:-staging}"
|
||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}"
|
||||
|
||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||
MASTER_KEY_B64="$("$PYTHON" -m govoplan_core.commands.config env-template --profile production-like --generate-secrets | sed -n 's/^MASTER_KEY_B64=//p' | head -n 1)"
|
||||
export MASTER_KEY_B64
|
||||
fi
|
||||
|
||||
compose() {
|
||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||
}
|
||||
|
||||
validate_config() {
|
||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
||||
}
|
||||
|
||||
case "$command" in
|
||||
start)
|
||||
exec "$ROOT/scripts/launch-production-like-dev.sh"
|
||||
;;
|
||||
stop)
|
||||
compose down
|
||||
;;
|
||||
status)
|
||||
compose ps
|
||||
validate_config
|
||||
;;
|
||||
seed)
|
||||
validate_config
|
||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||
;;
|
||||
reset)
|
||||
[ "$yes" = "1" ] || fail "reset is destructive; rerun with --yes"
|
||||
compose down -v
|
||||
rm -rf "$ROOT/runtime/production-like"
|
||||
;;
|
||||
validate-config)
|
||||
validate_config
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
fail "unknown command: $command"
|
||||
;;
|
||||
esac
|
||||
@@ -1,259 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
|
||||
|
||||
Generates a signed module package catalog for a GovOPlaN release, writes it into
|
||||
govoplan-web/public/catalogs/v1, validates it, optionally builds the website,
|
||||
and optionally commits/tags/pushes govoplan-web.
|
||||
|
||||
Options:
|
||||
--version <x.y.z> Release version without leading v.
|
||||
--channel <name> Catalog channel. Defaults to stable.
|
||||
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
||||
--expires-days <days> Catalog expiry window. Defaults to 90.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Ed25519 private key. May be repeated for rotation.
|
||||
--core-root <path> govoplan-core checkout. Defaults to this script parent.
|
||||
--web-root <path> govoplan-web checkout. Defaults to ../govoplan-web.
|
||||
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
|
||||
--npm <path> npm executable used for optional web build.
|
||||
--build-web Run npm run build in govoplan-web after generating files.
|
||||
--commit Commit generated catalog files in govoplan-web.
|
||||
--tag Tag govoplan-web with catalog-v<x.y.z>. Implies --commit.
|
||||
--push Push govoplan-web branch and tag. Implies --commit.
|
||||
--remote <name> Git remote for push. Defaults to origin.
|
||||
--branch <name> Branch to push. Defaults to current govoplan-web branch.
|
||||
-n, --dry-run Print commands and validate inputs without writing git changes.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
CORE_ROOT="$ROOT"
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
VERSION=""
|
||||
CHANNEL="stable"
|
||||
SEQUENCE=""
|
||||
EXPIRES_DAYS="90"
|
||||
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
BUILD_WEB=0
|
||||
COMMIT=0
|
||||
TAG_WEB=0
|
||||
PUSH=0
|
||||
DRY_RUN=0
|
||||
SIGNING_KEYS=()
|
||||
NPM_BIN="${NPM:-}"
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
VERSION="${2#v}"
|
||||
shift 2
|
||||
;;
|
||||
--channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--core-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CORE_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$(cd "$2" && pwd)"
|
||||
shift 2
|
||||
;;
|
||||
--public-base-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
PUBLIC_BASE_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web)
|
||||
BUILD_WEB=1
|
||||
shift
|
||||
;;
|
||||
--commit)
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--tag)
|
||||
TAG_WEB=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--push)
|
||||
PUSH=1
|
||||
COMMIT=1
|
||||
shift
|
||||
;;
|
||||
--remote)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--branch)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$VERSION" ]] || fail "--version is required"
|
||||
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
||||
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
||||
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
||||
fi
|
||||
|
||||
if [[ -z "$BRANCH" ]]; then
|
||||
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
||||
fi
|
||||
[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch"
|
||||
|
||||
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
||||
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
||||
TAG_NAME="catalog-v$VERSION"
|
||||
|
||||
run() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
GEN_ARGS=(
|
||||
"$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py"
|
||||
--version "$VERSION"
|
||||
--channel "$CHANNEL"
|
||||
--expires-days "$EXPIRES_DAYS"
|
||||
--catalog-output "$CATALOG_PATH"
|
||||
--keyring-output "$KEYRING_PATH"
|
||||
--public-base-url "$PUBLIC_BASE_URL"
|
||||
)
|
||||
if [[ -n "$SEQUENCE" ]]; then
|
||||
GEN_ARGS+=(--sequence "$SEQUENCE")
|
||||
fi
|
||||
for signing_key in "${SIGNING_KEYS[@]}"; do
|
||||
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
|
||||
run "${GEN_ARGS[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
|
||||
"$PYTHON" -m govoplan_core.commands.module_installer \
|
||||
--validate-package-catalog "$CATALOG_PATH" \
|
||||
--require-signed-catalog \
|
||||
--approved-catalog-channel "$CHANNEL" \
|
||||
--format json >/dev/null
|
||||
else
|
||||
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
|
||||
fi
|
||||
|
||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
|
||||
fi
|
||||
|
||||
if [[ "$COMMIT" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$WEB_ROOT" diff --cached --quiet; then
|
||||
echo "No govoplan-web catalog changes to commit."
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
else
|
||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
|
||||
fail "govoplan-web tag already exists: $TAG_NAME"
|
||||
fi
|
||||
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
||||
fi
|
||||
|
||||
if [[ "$PUSH" -eq 1 ]]; then
|
||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
||||
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
|
||||
else
|
||||
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Catalog ready:"
|
||||
echo " $CATALOG_PATH"
|
||||
echo " $KEYRING_PATH"
|
||||
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"
|
||||
@@ -1,861 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/push-release-tag.sh [options]
|
||||
|
||||
Bumps installable GovOPlaN package versions, commits pending changes in the
|
||||
GovOPlaN release/module repositories, creates an annotated release tag in each
|
||||
repo, and pushes branch+tag.
|
||||
|
||||
By default the script asks which version part to bump:
|
||||
major X.Y.Z -> (X+1).0.0
|
||||
minor X.Y.Z -> X.(Y+1).0
|
||||
subversion X.Y.Z -> X.Y.(Z+1)
|
||||
|
||||
Options:
|
||||
--bump <kind> Version bump: major, minor, subversion, or patch.
|
||||
--version <x.y.z> Explicit target version instead of a bump. May equal
|
||||
the current repo versions when they were bumped already.
|
||||
-r, --remote <name> Remote to push to. Defaults to origin.
|
||||
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
|
||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||
--warn-migration-audit Run the migration release audit without baseline strictness.
|
||||
--strict-migration-audit
|
||||
Require current Alembic heads to be recorded in the
|
||||
latest migration release baseline.
|
||||
--skip-migration-audit Do not run the release migration audit.
|
||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||
catalog into govoplan-web.
|
||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||
Defaults to ../govoplan-web.
|
||||
--catalog-signing-key <key-id=/path/private.pem>
|
||||
Catalog signing key for --publish-web-catalog. May be
|
||||
repeated during key rotation.
|
||||
--catalog-channel <name>
|
||||
Catalog channel. Defaults to stable.
|
||||
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
|
||||
--catalog-expires-days <days>
|
||||
Catalog expiry window. Defaults to 90.
|
||||
--catalog-public-url <url>
|
||||
Public website URL. Defaults to
|
||||
https://govoplan.add-ideas.de.
|
||||
--build-web-catalog Run npm run build in govoplan-web before committing the
|
||||
catalog update.
|
||||
--npm <path> npm executable for lockfile generation.
|
||||
-y, --yes Do not prompt before committing, tagging, and pushing.
|
||||
-n, --dry-run Print intended actions without changing files or git state.
|
||||
-h, --help Show this help.
|
||||
|
||||
Repos:
|
||||
Installable release packages:
|
||||
govoplan-access
|
||||
govoplan-admin
|
||||
govoplan-tenancy
|
||||
govoplan-organizations
|
||||
govoplan-identity
|
||||
govoplan-idm
|
||||
govoplan-policy
|
||||
govoplan-audit
|
||||
govoplan-dashboard
|
||||
govoplan-files
|
||||
govoplan-mail
|
||||
govoplan-campaign
|
||||
govoplan-calendar
|
||||
govoplan-ops
|
||||
govoplan-core
|
||||
|
||||
Roadmap/scaffold module repositories are tag-only until they have package
|
||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||
xrechnung, and xta-osci.
|
||||
|
||||
Examples:
|
||||
scripts/push-release-tag.sh
|
||||
scripts/push-release-tag.sh --bump subversion
|
||||
scripts/push-release-tag.sh --version 0.2.0 --message "Release v0.2.0"
|
||||
USAGE
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PARENT="$(dirname "$ROOT")"
|
||||
PACKAGE_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-access"
|
||||
"$PARENT/govoplan-admin"
|
||||
"$PARENT/govoplan-tenancy"
|
||||
"$PARENT/govoplan-organizations"
|
||||
"$PARENT/govoplan-identity"
|
||||
"$PARENT/govoplan-idm"
|
||||
"$PARENT/govoplan-policy"
|
||||
"$PARENT/govoplan-audit"
|
||||
"$PARENT/govoplan-dashboard"
|
||||
"$PARENT/govoplan-files"
|
||||
"$PARENT/govoplan-mail"
|
||||
"$PARENT/govoplan-campaign"
|
||||
"$PARENT/govoplan-calendar"
|
||||
"$PARENT/govoplan-docs"
|
||||
"$PARENT/govoplan-ops"
|
||||
)
|
||||
TAG_ONLY_MODULE_REPOS=(
|
||||
"$PARENT/govoplan-addresses"
|
||||
"$PARENT/govoplan-appointments"
|
||||
"$PARENT/govoplan-cases"
|
||||
"$PARENT/govoplan-connectors"
|
||||
"$PARENT/govoplan-dms"
|
||||
"$PARENT/govoplan-erp"
|
||||
"$PARENT/govoplan-fit-connect"
|
||||
"$PARENT/govoplan-forms"
|
||||
"$PARENT/govoplan-identity-trust"
|
||||
"$PARENT/govoplan-ledger"
|
||||
"$PARENT/govoplan-notifications"
|
||||
"$PARENT/govoplan-payments"
|
||||
"$PARENT/govoplan-portal"
|
||||
"$PARENT/govoplan-reporting"
|
||||
"$PARENT/govoplan-scheduling"
|
||||
"$PARENT/govoplan-search"
|
||||
"$PARENT/govoplan-tasks"
|
||||
"$PARENT/govoplan-templates"
|
||||
"$PARENT/govoplan-workflow"
|
||||
"$PARENT/govoplan-xoev"
|
||||
"$PARENT/govoplan-xrechnung"
|
||||
"$PARENT/govoplan-xta-osci"
|
||||
)
|
||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
||||
REPOS=(
|
||||
"${MODULE_REPOS[@]}"
|
||||
"$ROOT"
|
||||
)
|
||||
|
||||
REMOTE="origin"
|
||||
BRANCH=""
|
||||
BUMP=""
|
||||
TARGET_VERSION=""
|
||||
COMMIT_MESSAGE=""
|
||||
TAG_MESSAGE=""
|
||||
DRY_RUN=0
|
||||
YES=0
|
||||
GENERATE_RELEASE_LOCK=1
|
||||
MIGRATION_AUDIT_MODE="auto"
|
||||
NPM_BIN="${NPM:-}"
|
||||
PUBLISH_WEB_CATALOG=0
|
||||
WEB_ROOT="$PARENT/govoplan-web"
|
||||
CATALOG_CHANNEL="stable"
|
||||
CATALOG_SEQUENCE=""
|
||||
CATALOG_EXPIRES_DAYS="90"
|
||||
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
|
||||
BUILD_WEB_CATALOG=0
|
||||
CATALOG_SIGNING_KEYS=()
|
||||
|
||||
if [[ -z "${PYTHON:-}" ]]; then
|
||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
||||
PYTHON="$ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_BIN" ]]; then
|
||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
||||
else
|
||||
NPM_BIN="npm"
|
||||
fi
|
||||
fi
|
||||
|
||||
normalize_bump() {
|
||||
local value="${1,,}"
|
||||
case "$value" in
|
||||
major|minor|subversion)
|
||||
printf '%s\n' "$value"
|
||||
;;
|
||||
patch|sub|sub-version)
|
||||
printf '%s\n' "subversion"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_version() {
|
||||
[[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]
|
||||
}
|
||||
|
||||
version_gt() {
|
||||
local new_major new_minor new_patch old_major old_minor old_patch
|
||||
IFS=. read -r new_major new_minor new_patch <<< "$1"
|
||||
IFS=. read -r old_major old_minor old_patch <<< "$2"
|
||||
|
||||
(( new_major > old_major )) && return 0
|
||||
(( new_major < old_major )) && return 1
|
||||
(( new_minor > old_minor )) && return 0
|
||||
(( new_minor < old_minor )) && return 1
|
||||
(( new_patch > old_patch )) && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
bump_version() {
|
||||
local version="$1"
|
||||
local bump="$2"
|
||||
local major minor patch
|
||||
IFS=. read -r major minor patch <<< "$version"
|
||||
|
||||
case "$bump" in
|
||||
major)
|
||||
printf '%s.0.0\n' "$((major + 1))"
|
||||
;;
|
||||
minor)
|
||||
printf '%s.%s.0\n' "$major" "$((minor + 1))"
|
||||
;;
|
||||
subversion)
|
||||
printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))"
|
||||
;;
|
||||
*)
|
||||
fail "unknown bump kind: $bump"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
get_project_name() {
|
||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["name"])' "$1/pyproject.toml"
|
||||
}
|
||||
|
||||
get_project_version() {
|
||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$1/pyproject.toml"
|
||||
}
|
||||
|
||||
update_pyproject() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
|
||||
"$PYTHON" - "$repo/pyproject.toml" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
text = path.read_text()
|
||||
data = tomllib.loads(text)
|
||||
project = data["project"]
|
||||
name = project["name"]
|
||||
old_version = project["version"]
|
||||
|
||||
text, version_count = re.subn(
|
||||
r'(?m)^version\s*=\s*["\'][^"\']+["\']\s*$',
|
||||
f'version = "{new_version}"',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if version_count != 1:
|
||||
raise SystemExit(f"could not update project.version in {path}")
|
||||
|
||||
if name != "govoplan-core":
|
||||
text, dependency_count = re.subn(
|
||||
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
|
||||
rf'\g<1>{new_version}',
|
||||
text,
|
||||
)
|
||||
if dependency_count < 1:
|
||||
raise SystemExit(f"could not update govoplan-core dependency in {path}")
|
||||
|
||||
path.write_text(text)
|
||||
print(f"{name}: {old_version} -> {new_version}")
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_manifest_version() {
|
||||
local repo="$1"
|
||||
local project_name="$2"
|
||||
local version="$3"
|
||||
local manifest_path=""
|
||||
|
||||
case "$project_name" in
|
||||
govoplan-core)
|
||||
return 0
|
||||
;;
|
||||
govoplan-access)
|
||||
manifest_path="$repo/src/govoplan_access/backend/manifest.py"
|
||||
;;
|
||||
govoplan-admin)
|
||||
manifest_path="$repo/src/govoplan_admin/backend/manifest.py"
|
||||
;;
|
||||
govoplan-tenancy)
|
||||
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
||||
;;
|
||||
govoplan-organizations)
|
||||
manifest_path="$repo/src/govoplan_organizations/backend/manifest.py"
|
||||
;;
|
||||
govoplan-identity)
|
||||
manifest_path="$repo/src/govoplan_identity/backend/manifest.py"
|
||||
;;
|
||||
govoplan-idm)
|
||||
manifest_path="$repo/src/govoplan_idm/backend/manifest.py"
|
||||
;;
|
||||
govoplan-policy)
|
||||
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
||||
;;
|
||||
govoplan-audit)
|
||||
manifest_path="$repo/src/govoplan_audit/backend/manifest.py"
|
||||
;;
|
||||
govoplan-files)
|
||||
manifest_path="$repo/src/govoplan_files/backend/manifest.py"
|
||||
;;
|
||||
govoplan-mail)
|
||||
manifest_path="$repo/src/govoplan_mail/backend/manifest.py"
|
||||
;;
|
||||
govoplan-campaign)
|
||||
manifest_path="$repo/src/govoplan_campaign/backend/manifest.py"
|
||||
;;
|
||||
govoplan-calendar)
|
||||
manifest_path="$repo/src/govoplan_calendar/backend/manifest.py"
|
||||
;;
|
||||
*)
|
||||
fail "unknown project for manifest version update: $project_name"
|
||||
;;
|
||||
esac
|
||||
|
||||
[[ -f "$manifest_path" ]] || fail "missing module manifest: $manifest_path"
|
||||
|
||||
"$PYTHON" - "$manifest_path" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
text = path.read_text()
|
||||
text, count = re.subn(
|
||||
r'(?m)^(\s*version=)["\'][^"\']+["\'](,?\s*)$',
|
||||
rf'\1"{new_version}"\2',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if count != 1:
|
||||
raise SystemExit(f"could not update ModuleManifest.version in {path}")
|
||||
path.write_text(text)
|
||||
PYCODE
|
||||
}
|
||||
|
||||
update_webui_package() {
|
||||
local repo="$1"
|
||||
local project_name="$2"
|
||||
local version="$3"
|
||||
local package_path=""
|
||||
local release_package_path="$repo/webui/package.release.json"
|
||||
|
||||
for package_path in "$repo/package.json" "$repo/webui/package.json"; do
|
||||
[[ -f "$package_path" ]] || continue
|
||||
"$PYTHON" - "$package_path" "$project_name" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
project_name = sys.argv[2]
|
||||
new_version = sys.argv[3]
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
data["version"] = new_version
|
||||
if project_name != "govoplan-core":
|
||||
peers = data.setdefault("peerDependencies", {})
|
||||
if "@govoplan/core-webui" in peers:
|
||||
peers["@govoplan/core-webui"] = f"^{new_version}"
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
done
|
||||
|
||||
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
||||
"$PYTHON" - "$release_package_path" "$version" <<'PYCODE'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
new_version = sys.argv[2]
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
data["version"] = new_version
|
||||
dependencies = data.setdefault("dependencies", {})
|
||||
for name, spec in list(dependencies.items()):
|
||||
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
|
||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
PYCODE
|
||||
fi
|
||||
}
|
||||
|
||||
update_version_files() {
|
||||
local repo="$1"
|
||||
local version="$2"
|
||||
local project_name="${PROJECT_NAMES[$repo]}"
|
||||
|
||||
update_pyproject "$repo" "$version"
|
||||
update_manifest_version "$repo" "$project_name" "$version"
|
||||
update_webui_package "$repo" "$project_name" "$version"
|
||||
}
|
||||
|
||||
refresh_development_webui_lock() {
|
||||
[[ -f "$ROOT/webui/package.json" ]] || return 0
|
||||
[[ -f "$ROOT/webui/package-lock.json" ]] || return 0
|
||||
|
||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$ROOT/webui" install --package-lock-only --ignore-scripts
|
||||
}
|
||||
|
||||
generate_release_lock() {
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 0 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
[[ -x "$ROOT/scripts/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $ROOT/scripts/generate-release-lock.sh"
|
||||
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
||||
}
|
||||
|
||||
run_migration_release_audit() {
|
||||
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
||||
local command=("$PYTHON" "$audit_script")
|
||||
|
||||
case "$MIGRATION_AUDIT_MODE" in
|
||||
skip)
|
||||
echo "Migration release audit: skipped"
|
||||
return
|
||||
;;
|
||||
strict)
|
||||
command+=("--strict")
|
||||
;;
|
||||
auto)
|
||||
command+=("--strict-if-baseline")
|
||||
;;
|
||||
warn)
|
||||
;;
|
||||
*)
|
||||
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
||||
;;
|
||||
esac
|
||||
|
||||
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
||||
run "${command[@]}"
|
||||
}
|
||||
|
||||
print_command() {
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
run() {
|
||||
print_command "$@"
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_for_bump() {
|
||||
local answer=""
|
||||
|
||||
if [[ -n "$BUMP" || -n "$TARGET_VERSION" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
fail "pass --bump major|minor|subversion or --version x.y.z for non-interactive use"
|
||||
fi
|
||||
|
||||
echo "Select version increase:"
|
||||
echo " 1) major X.Y.Z -> (X+1).0.0"
|
||||
echo " 2) minor X.Y.Z -> X.(Y+1).0"
|
||||
echo " 3) subversion X.Y.Z -> X.Y.(Z+1)"
|
||||
printf 'Choice [subversion]: '
|
||||
read -r answer
|
||||
|
||||
case "${answer,,}" in
|
||||
""|3|subversion|patch|sub|sub-version)
|
||||
BUMP="subversion"
|
||||
;;
|
||||
1|major)
|
||||
BUMP="major"
|
||||
;;
|
||||
2|minor)
|
||||
BUMP="minor"
|
||||
;;
|
||||
*)
|
||||
fail "unknown version bump: $answer"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
confirm_release() {
|
||||
local answer=""
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 || "$YES" -eq 1 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
fail "refusing to commit, tag, and push without confirmation; pass --yes for non-interactive use"
|
||||
fi
|
||||
|
||||
printf 'Commit all changes, tag all repos, and push to %s? [y/N] ' "$REMOTE"
|
||||
read -r answer
|
||||
case "${answer,,}" in
|
||||
y|yes)
|
||||
;;
|
||||
*)
|
||||
echo "Aborted."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--bump)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BUMP="$(normalize_bump "$2")" || fail "unknown bump kind: $2"
|
||||
shift 2
|
||||
;;
|
||||
--version)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
TARGET_VERSION="$2"
|
||||
shift 2
|
||||
;;
|
||||
-r|--remote)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-b|--branch)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-m|--message)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
COMMIT_MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--tag-message)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
TAG_MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--skip-release-lock)
|
||||
GENERATE_RELEASE_LOCK=0
|
||||
shift
|
||||
;;
|
||||
--warn-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="warn"
|
||||
shift
|
||||
;;
|
||||
--strict-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="strict"
|
||||
shift
|
||||
;;
|
||||
--skip-migration-audit)
|
||||
MIGRATION_AUDIT_MODE="skip"
|
||||
shift
|
||||
;;
|
||||
--publish-web-catalog)
|
||||
PUBLISH_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--web-root)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
WEB_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-signing-key)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SIGNING_KEYS+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--catalog-channel)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_CHANNEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-sequence)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_SEQUENCE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-expires-days)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_EXPIRES_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--catalog-public-url)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
CATALOG_PUBLIC_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--build-web-catalog)
|
||||
BUILD_WEB_CATALOG=1
|
||||
shift
|
||||
;;
|
||||
--npm)
|
||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
||||
NPM_BIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-y|--yes)
|
||||
YES=1
|
||||
shift
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
||||
fail "use either --bump or --version, not both"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
|
||||
fail "--publish-web-catalog requires at least one --catalog-signing-key"
|
||||
fi
|
||||
|
||||
prompt_for_bump
|
||||
|
||||
if ! command -v "$PYTHON" >/dev/null 2>&1; then
|
||||
fail "Python interpreter not found: $PYTHON"
|
||||
fi
|
||||
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
|
||||
fail "npm executable not found: $NPM_BIN"
|
||||
fi
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
[[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT"
|
||||
[[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh"
|
||||
fi
|
||||
|
||||
declare -A PROJECT_NAMES
|
||||
declare -A OLD_VERSIONS
|
||||
declare -A BRANCHES
|
||||
declare -A REPO_KINDS
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
||||
|
||||
if [[ -f "$repo/pyproject.toml" ]]; then
|
||||
REPO_KINDS["$repo"]="package"
|
||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
||||
|
||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
||||
else
|
||||
REPO_KINDS["$repo"]="tag-only"
|
||||
PROJECT_NAMES["$repo"]="$(basename "$repo")"
|
||||
OLD_VERSIONS["$repo"]="tag-only"
|
||||
fi
|
||||
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
BRANCHES["$repo"]="$BRANCH"
|
||||
else
|
||||
BRANCHES["$repo"]="$(git -C "$repo" symbolic-ref --quiet --short HEAD || true)"
|
||||
fi
|
||||
|
||||
[[ -n "${BRANCHES[$repo]}" ]] || fail "cannot infer branch for $repo; pass --branch <name>"
|
||||
git -C "$repo" check-ref-format "refs/heads/${BRANCHES[$repo]}" || fail "invalid branch name ${BRANCHES[$repo]} for $repo"
|
||||
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
||||
|
||||
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
||||
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
|
||||
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
||||
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$TARGET_VERSION" ]]; then
|
||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
||||
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
||||
fi
|
||||
done
|
||||
TARGET_VERSION="$(bump_version "$BASE_VERSION" "$BUMP")"
|
||||
fi
|
||||
|
||||
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
||||
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
|
||||
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
|
||||
fi
|
||||
done
|
||||
|
||||
TAG="v$TARGET_VERSION"
|
||||
COMMIT_MESSAGE="${COMMIT_MESSAGE:-Release $TAG}"
|
||||
TAG_MESSAGE="${TAG_MESSAGE:-$COMMIT_MESSAGE}"
|
||||
|
||||
git -C "$ROOT" check-ref-format "refs/tags/$TAG" || fail "invalid tag name: $TAG"
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
if git -C "$repo" rev-parse --quiet --verify "refs/tags/$TAG" >/dev/null; then
|
||||
fail "local tag already exists in ${PROJECT_NAMES[$repo]}: $TAG"
|
||||
fi
|
||||
|
||||
set +e
|
||||
git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
|
||||
ls_remote_status=$?
|
||||
set -e
|
||||
|
||||
if [[ "$ls_remote_status" -eq 0 ]]; then
|
||||
fail "remote tag already exists for ${PROJECT_NAMES[$repo]} on $REMOTE: $TAG"
|
||||
elif [[ "$ls_remote_status" -ne 2 ]]; then
|
||||
fail "could not check remote tags for ${PROJECT_NAMES[$repo]} on $REMOTE"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Release plan:"
|
||||
echo " version: $TARGET_VERSION"
|
||||
echo " tag: $TAG"
|
||||
echo " remote: $REMOTE"
|
||||
echo " commit message: $COMMIT_MESSAGE"
|
||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
||||
else
|
||||
echo " release lock: skipped"
|
||||
fi
|
||||
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||
else
|
||||
echo " web catalog: skipped"
|
||||
fi
|
||||
echo
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
||||
else
|
||||
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
|
||||
fi
|
||||
git -C "$repo" status --short
|
||||
echo
|
||||
done
|
||||
|
||||
run_migration_release_audit
|
||||
|
||||
confirm_release
|
||||
|
||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Would update version files in $repo to $TARGET_VERSION"
|
||||
else
|
||||
update_version_files "$repo" "$TARGET_VERSION"
|
||||
fi
|
||||
done
|
||||
|
||||
refresh_development_webui_lock
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$repo" diff --cached --quiet; then
|
||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
||||
fi
|
||||
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
else
|
||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
||||
fi
|
||||
|
||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
done
|
||||
|
||||
for repo in "${MODULE_REPOS[@]}"; do
|
||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
||||
done
|
||||
|
||||
generate_release_lock
|
||||
|
||||
run git -C "$ROOT" add -A
|
||||
|
||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
||||
if git -C "$ROOT" diff --cached --quiet; then
|
||||
fail "no staged changes for ${PROJECT_NAMES[$ROOT]} after version bump"
|
||||
fi
|
||||
fi
|
||||
|
||||
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
||||
|
||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS=(
|
||||
"$ROOT/scripts/publish-release-catalog.sh"
|
||||
--version "$TARGET_VERSION"
|
||||
--channel "$CATALOG_CHANNEL"
|
||||
--expires-days "$CATALOG_EXPIRES_DAYS"
|
||||
--core-root "$ROOT"
|
||||
--web-root "$WEB_ROOT"
|
||||
--public-base-url "$CATALOG_PUBLIC_URL"
|
||||
--npm "$NPM_BIN"
|
||||
--commit
|
||||
--tag
|
||||
--push
|
||||
--remote "$REMOTE"
|
||||
)
|
||||
if [[ -n "$BRANCH" ]]; then
|
||||
CATALOG_ARGS+=(--branch "$BRANCH")
|
||||
fi
|
||||
if [[ -n "$CATALOG_SEQUENCE" ]]; then
|
||||
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
|
||||
fi
|
||||
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--build-web)
|
||||
fi
|
||||
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
|
||||
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
|
||||
done
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
CATALOG_ARGS+=(--dry-run)
|
||||
fi
|
||||
run "${CATALOG_ARGS[@]}"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
||||
else
|
||||
echo "Released $TAG across all GovOPlaN repos."
|
||||
fi
|
||||
@@ -1,422 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Migration:
|
||||
owner: str
|
||||
path: Path
|
||||
revision: str
|
||||
down_revisions: tuple[str, ...]
|
||||
branch_labels: tuple[str, ...]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workspace-root",
|
||||
type=Path,
|
||||
default=ROOT.parent,
|
||||
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-file",
|
||||
type=Path,
|
||||
default=DEFAULT_BASELINE_FILE,
|
||||
help="JSON file recording released migration heads.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Fail when current heads are not recorded in the latest release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-if-baseline",
|
||||
action="store_true",
|
||||
help="Run in strict mode only after at least one release baseline is recorded.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--record-release",
|
||||
metavar="VERSION",
|
||||
help="Record the current reviewed migration heads as a release baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--replace-release",
|
||||
action="store_true",
|
||||
help="Replace an existing release entry when used with --record-release.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--squash-plan",
|
||||
action="store_true",
|
||||
help="Print the reviewed/manual squash checklist for the current graph.",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||
args = parser.parse_args()
|
||||
|
||||
migrations = discover_migrations(args.workspace_root)
|
||||
baseline = load_baseline_file(args.baseline_file)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.record_release:
|
||||
if report["graph_errors"]:
|
||||
print_text_report(report)
|
||||
return 1
|
||||
baseline = record_release_baseline(
|
||||
baseline,
|
||||
report,
|
||||
release=args.record_release,
|
||||
replace=args.replace_release,
|
||||
)
|
||||
write_baseline_file(args.baseline_file, baseline)
|
||||
report = build_report(
|
||||
migrations,
|
||||
baseline=baseline,
|
||||
baseline_file=args.baseline_file,
|
||||
workspace_root=args.workspace_root,
|
||||
)
|
||||
|
||||
if args.squash_plan:
|
||||
print_squash_plan(report)
|
||||
elif args.json:
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
else:
|
||||
print_text_report(report)
|
||||
|
||||
has_graph_errors = bool(report["graph_errors"])
|
||||
has_strict_errors = bool(report["strict_errors"])
|
||||
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
||||
if has_graph_errors or (strict_required and has_strict_errors):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
||||
migrations: list[Migration] = []
|
||||
for versions_dir in _migration_version_dirs(workspace_root):
|
||||
owner = owner_for_versions_dir(versions_dir)
|
||||
for path in sorted(versions_dir.glob("*.py")):
|
||||
if path.name == "__init__.py":
|
||||
continue
|
||||
migration = parse_migration_file(owner, path)
|
||||
if migration is not None:
|
||||
migrations.append(migration)
|
||||
return migrations
|
||||
|
||||
|
||||
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
||||
candidates = [ROOT / "alembic" / "versions"]
|
||||
for repo in sorted(workspace_root.glob("govoplan-*")):
|
||||
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
||||
seen: set[Path] = set()
|
||||
result: list[Path] = []
|
||||
for candidate in candidates:
|
||||
resolved = candidate.resolve()
|
||||
if resolved in seen or not candidate.is_dir():
|
||||
continue
|
||||
seen.add(resolved)
|
||||
result.append(candidate)
|
||||
return result
|
||||
|
||||
|
||||
def owner_for_versions_dir(versions_dir: Path) -> str:
|
||||
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
||||
return "govoplan-core"
|
||||
for parent in versions_dir.parents:
|
||||
if parent.name.startswith("govoplan-"):
|
||||
return parent.name
|
||||
return versions_dir.parent.name
|
||||
|
||||
|
||||
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
values: dict[str, Any] = {}
|
||||
for statement in tree.body:
|
||||
if isinstance(statement, ast.Assign):
|
||||
for target in statement.targets:
|
||||
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
||||
values[target.id] = ast.literal_eval(statement.value)
|
||||
elif (
|
||||
isinstance(statement, ast.AnnAssign)
|
||||
and isinstance(statement.target, ast.Name)
|
||||
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
||||
and statement.value is not None
|
||||
):
|
||||
values[statement.target.id] = ast.literal_eval(statement.value)
|
||||
revision = values.get("revision")
|
||||
if not isinstance(revision, str):
|
||||
return None
|
||||
return Migration(
|
||||
owner=owner,
|
||||
path=path,
|
||||
revision=revision,
|
||||
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
||||
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value if item is not None)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return tuple(str(item) for item in value)
|
||||
return (str(value),)
|
||||
|
||||
|
||||
def load_baseline_file(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"version": 1, "releases": [], "_missing": True}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit(f"{path} must contain a JSON object")
|
||||
releases = payload.get("releases")
|
||||
if not isinstance(releases, list):
|
||||
raise SystemExit(f"{path} must contain a releases list")
|
||||
return payload
|
||||
|
||||
|
||||
def build_report(
|
||||
migrations: list[Migration],
|
||||
*,
|
||||
baseline: dict[str, Any],
|
||||
baseline_file: Path,
|
||||
workspace_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
revisions: dict[str, Migration] = {}
|
||||
graph_errors: list[str] = []
|
||||
for migration in migrations:
|
||||
existing = revisions.get(migration.revision)
|
||||
if existing is not None:
|
||||
graph_errors.append(
|
||||
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
||||
)
|
||||
revisions[migration.revision] = migration
|
||||
|
||||
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
||||
for migration in migrations:
|
||||
for down_revision in migration.down_revisions:
|
||||
if down_revision not in revisions:
|
||||
graph_errors.append(
|
||||
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
||||
)
|
||||
|
||||
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
||||
current_head_entries = [
|
||||
{
|
||||
"owner": revisions[revision].owner,
|
||||
"revision": revision,
|
||||
}
|
||||
for revision in current_heads
|
||||
]
|
||||
owner_rows = []
|
||||
for owner in sorted({migration.owner for migration in migrations}):
|
||||
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
||||
owner_revisions = {migration.revision for migration in owner_migrations}
|
||||
owner_referenced = {
|
||||
revision
|
||||
for migration in owner_migrations
|
||||
for revision in migration.down_revisions
|
||||
if revision in owner_revisions
|
||||
}
|
||||
owner_heads = sorted(owner_revisions - owner_referenced)
|
||||
owner_rows.append(
|
||||
{
|
||||
"owner": owner,
|
||||
"migrations": len(owner_migrations),
|
||||
"heads": owner_heads,
|
||||
}
|
||||
)
|
||||
|
||||
releases = baseline.get("releases") or []
|
||||
latest_release = releases[-1] if releases else None
|
||||
latest_heads = _latest_release_heads(latest_release)
|
||||
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
||||
|
||||
strict_errors: list[str] = []
|
||||
if baseline.get("_missing"):
|
||||
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
||||
if not releases:
|
||||
strict_errors.append("no released migration baseline has been recorded yet")
|
||||
if unrecorded_heads:
|
||||
strict_errors.append(
|
||||
"current migration heads are not recorded in the latest release baseline: "
|
||||
+ ", ".join(unrecorded_heads)
|
||||
)
|
||||
return {
|
||||
"workspace_root": str(workspace_root),
|
||||
"baseline_file": str(baseline_file),
|
||||
"baseline_file_missing": bool(baseline.get("_missing")),
|
||||
"release_count": len(releases),
|
||||
"latest_release": latest_release,
|
||||
"owners": owner_rows,
|
||||
"current_heads": current_heads,
|
||||
"current_head_entries": current_head_entries,
|
||||
"graph_errors": graph_errors,
|
||||
"strict_errors": strict_errors,
|
||||
}
|
||||
|
||||
|
||||
def _latest_release_heads(latest_release: Any) -> set[str]:
|
||||
if not isinstance(latest_release, dict):
|
||||
return set()
|
||||
heads = latest_release.get("heads") or []
|
||||
graph_heads = latest_release.get("graph_heads") or []
|
||||
result: set[str] = set()
|
||||
for head_list in (heads, graph_heads):
|
||||
if not isinstance(head_list, list):
|
||||
continue
|
||||
for entry in head_list:
|
||||
if isinstance(entry, str):
|
||||
result.add(entry)
|
||||
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
||||
result.add(entry["revision"])
|
||||
owner_heads = latest_release.get("owner_heads") or []
|
||||
if isinstance(owner_heads, list):
|
||||
for entry in owner_heads:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
revisions = entry.get("revisions") or []
|
||||
if isinstance(revisions, str):
|
||||
result.add(revisions)
|
||||
elif isinstance(revisions, list):
|
||||
result.update(revision for revision in revisions if isinstance(revision, str))
|
||||
return result
|
||||
|
||||
|
||||
def record_release_baseline(
|
||||
baseline: dict[str, Any],
|
||||
report: dict[str, Any],
|
||||
*,
|
||||
release: str,
|
||||
replace: bool,
|
||||
) -> dict[str, Any]:
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in baseline.items()
|
||||
if not key.startswith("_")
|
||||
}
|
||||
payload.setdefault("version", 1)
|
||||
releases = list(payload.get("releases") or [])
|
||||
existing_index = next(
|
||||
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
||||
None,
|
||||
)
|
||||
if existing_index is not None and not replace:
|
||||
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
||||
|
||||
entry = {
|
||||
"release": release,
|
||||
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"squash_policy": "reviewed-manual",
|
||||
"heads": report["current_head_entries"],
|
||||
"owner_heads": [
|
||||
{
|
||||
"owner": owner["owner"],
|
||||
"revisions": owner["heads"],
|
||||
}
|
||||
for owner in report["owners"]
|
||||
],
|
||||
}
|
||||
if existing_index is None:
|
||||
releases.append(entry)
|
||||
else:
|
||||
releases[existing_index] = entry
|
||||
payload["releases"] = releases
|
||||
return payload
|
||||
|
||||
|
||||
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def print_text_report(report: dict[str, Any]) -> None:
|
||||
print("Migration release audit")
|
||||
print(f" workspace: {report['workspace_root']}")
|
||||
print(f" baseline file: {report['baseline_file']}")
|
||||
print(f" recorded releases: {report['release_count']}")
|
||||
print()
|
||||
print("Owners:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
||||
print()
|
||||
print("Current heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
|
||||
if report["graph_errors"]:
|
||||
print()
|
||||
print("Graph errors:")
|
||||
for error in report["graph_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
if report["strict_errors"]:
|
||||
print()
|
||||
print("Release baseline warnings:")
|
||||
for error in report["strict_errors"]:
|
||||
print(f" - {error}")
|
||||
|
||||
|
||||
def print_squash_plan(report: dict[str, Any]) -> None:
|
||||
print("Manual migration squash plan")
|
||||
print()
|
||||
print("Policy:")
|
||||
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
||||
print(" - Squash only unreleased development migrations.")
|
||||
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
||||
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
||||
print()
|
||||
print("Current owner heads:")
|
||||
for owner in report["owners"]:
|
||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||
print(f" - {owner['owner']}: {heads}")
|
||||
print()
|
||||
print("Current Alembic graph heads:")
|
||||
for entry in report["current_head_entries"]:
|
||||
print(f" - {entry['owner']}: {entry['revision']}")
|
||||
if not report["current_head_entries"]:
|
||||
print(" -")
|
||||
print()
|
||||
print("Release steps:")
|
||||
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
||||
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
||||
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
||||
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
||||
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -20,30 +20,13 @@ from govoplan_core.core.events import (
|
||||
)
|
||||
from govoplan_core.core.runtime import get_registry
|
||||
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
||||
from govoplan_core.security.redaction import redact_secret_values
|
||||
|
||||
|
||||
AUDIT_MODULE_ID = "audit"
|
||||
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
|
||||
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
|
||||
|
||||
SENSITIVE_DETAIL_KEYS = {
|
||||
"password",
|
||||
"smtp_password",
|
||||
"imap_password",
|
||||
"secret",
|
||||
"api_key",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"authorization",
|
||||
"cookie",
|
||||
"credentials",
|
||||
"credential",
|
||||
"private_key",
|
||||
"claim_token",
|
||||
"build_token",
|
||||
}
|
||||
|
||||
_ACTION_MODULE_PREFIXES = {
|
||||
"api_key": "access",
|
||||
"configuration_change": "access",
|
||||
@@ -90,17 +73,7 @@ def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]:
|
||||
|
||||
|
||||
def _sanitize_details(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).lower() in SENSITIVE_DETAIL_KEYS:
|
||||
sanitized[key] = "<redacted>"
|
||||
else:
|
||||
sanitized[key] = _sanitize_details(item)
|
||||
return sanitized
|
||||
if isinstance(value, list):
|
||||
return [_sanitize_details(item) for item in value]
|
||||
return value
|
||||
return redact_secret_values(value)
|
||||
|
||||
|
||||
def audit_operation_context(
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.db.bootstrap import bootstrap_dev_data
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
from govoplan_core.db.migrations import (
|
||||
ModuleMigrationTaskExecutionError,
|
||||
POST_MIGRATION_TASK_PHASES,
|
||||
PRE_MIGRATION_TASK_PHASES,
|
||||
migrate_database,
|
||||
run_registered_module_migration_tasks,
|
||||
)
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.settings import settings
|
||||
|
||||
@@ -11,12 +19,50 @@ from govoplan_core.settings import settings
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
|
||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
|
||||
parser.add_argument(
|
||||
"--migration-track",
|
||||
default=settings.migration_track,
|
||||
choices=("release", "dev"),
|
||||
help="Alembic migration track to load: release uses squashed release shortcuts; dev uses detailed development chains.",
|
||||
)
|
||||
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
|
||||
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
|
||||
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
|
||||
parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key")
|
||||
parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create")
|
||||
args = parser.parse_args()
|
||||
|
||||
configure_database(args.database_url)
|
||||
migration = migrate_database(database_url=args.database_url)
|
||||
enabled_modules = tuple(args.enabled_module) if args.enabled_module else None
|
||||
migration_order = tuple(args.migration_module) if args.migration_module else None
|
||||
task_records: list[dict[str, object]] = []
|
||||
try:
|
||||
_run_migration_tasks(
|
||||
task_records,
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_order=migration_order,
|
||||
phases=PRE_MIGRATION_TASK_PHASES,
|
||||
)
|
||||
migration = migrate_database(
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_module_order=migration_order,
|
||||
migration_track=args.migration_track,
|
||||
)
|
||||
_run_migration_tasks(
|
||||
task_records,
|
||||
database_url=args.database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_order=migration_order,
|
||||
phases=POST_MIGRATION_TASK_PHASES,
|
||||
)
|
||||
finally:
|
||||
if args.migration_task_record_output:
|
||||
args.migration_task_record_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.migration_task_record_output.write_text(json.dumps(task_records, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
if task_records:
|
||||
print(f"Executed {len(task_records)} module migration task(s).")
|
||||
if migration.reconciled_revision:
|
||||
print(f"Reconciled legacy database marker to {migration.reconciled_revision}.")
|
||||
print(f"Database schema upgraded to {migration.current_revision}.")
|
||||
@@ -33,5 +79,26 @@ def main() -> None:
|
||||
print("Development API key already exists or was not requested.")
|
||||
|
||||
|
||||
def _run_migration_tasks(
|
||||
target: list[dict[str, object]],
|
||||
*,
|
||||
database_url: str,
|
||||
enabled_modules: tuple[str, ...] | None,
|
||||
migration_order: tuple[str, ...] | None,
|
||||
phases: tuple[str, ...],
|
||||
) -> None:
|
||||
try:
|
||||
records = run_registered_module_migration_tasks(
|
||||
database_url=database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
migration_module_order=migration_order,
|
||||
phases=phases,
|
||||
)
|
||||
except ModuleMigrationTaskExecutionError as exc:
|
||||
target.extend(exc.records)
|
||||
raise
|
||||
target.extend(records)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -85,6 +85,9 @@ AccessProvenanceKind = Literal[
|
||||
"identity",
|
||||
"account",
|
||||
"tenant_membership",
|
||||
"resource",
|
||||
"owner",
|
||||
"share",
|
||||
"organization_unit",
|
||||
"function",
|
||||
"group",
|
||||
@@ -496,6 +499,20 @@ class ResourceAccessService(Protocol):
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ResourceAccessExplanationProvider(Protocol):
|
||||
def explain_resource_provenance(
|
||||
self,
|
||||
session: object,
|
||||
principal: PrincipalRef,
|
||||
*,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
action: str,
|
||||
) -> Sequence[AccessDecisionProvenance]:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class AccessExplanationService(Protocol):
|
||||
def explain_scope_provenance(
|
||||
|
||||
@@ -12,6 +12,7 @@ from govoplan_core.admin.settings import get_system_settings
|
||||
from govoplan_core.core.change_sequence import record_change
|
||||
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
|
||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||
from govoplan_core.security.redaction import redact_secret_values
|
||||
from govoplan_core.security.time import utc_now
|
||||
|
||||
|
||||
@@ -377,17 +378,7 @@ def _sanitize_value(key: str, value: object) -> object:
|
||||
|
||||
|
||||
def _redact_secrets(value: object) -> object:
|
||||
if isinstance(value, dict):
|
||||
result: dict[str, object] = {}
|
||||
for key, item in value.items():
|
||||
if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
|
||||
result[str(key)] = "<redacted>"
|
||||
else:
|
||||
result[str(key)] = _redact_secrets(item)
|
||||
return result
|
||||
if isinstance(value, list):
|
||||
return [_redact_secrets(item) for item in value]
|
||||
return value
|
||||
return redact_secret_values(value)
|
||||
|
||||
|
||||
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Mapping
|
||||
from typing import Any, Literal
|
||||
|
||||
from govoplan_core.security.permissions import scopes_grant
|
||||
from govoplan_core.security.redaction import contains_plain_secret
|
||||
|
||||
|
||||
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
|
||||
@@ -411,13 +412,4 @@ def _policy_explanation(field: ConfigurationFieldSafety) -> str:
|
||||
|
||||
|
||||
def _contains_plain_secret(value: object) -> bool:
|
||||
if not isinstance(value, Mapping):
|
||||
return False
|
||||
secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
|
||||
for key, item in value.items():
|
||||
name = str(key).strip().casefold()
|
||||
if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
|
||||
return True
|
||||
if isinstance(item, Mapping) and _contains_plain_secret(item):
|
||||
return True
|
||||
return False
|
||||
return contains_plain_secret(value)
|
||||
|
||||
13
src/govoplan_core/core/files.py
Normal file
13
src/govoplan_core/core/files.py
Normal file
@@ -0,0 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from govoplan_core.core.access import ResourceAccessExplanationProvider
|
||||
|
||||
|
||||
CAPABILITY_FILES_ACCESS = "files.access"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class FileAccessProvider(ResourceAccessExplanationProvider, Protocol):
|
||||
"""Resource-level access explanation provider for Files-owned resources."""
|
||||
@@ -12,6 +12,29 @@ from govoplan_core.core.registry import PlatformRegistry
|
||||
class MigrationMetadataPlan:
|
||||
metadata: tuple[MetaData, ...]
|
||||
script_locations: tuple[str, ...]
|
||||
modules: tuple[ModuleMigrationMetadata, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationMetadata:
|
||||
module_id: str
|
||||
script_location: str | None = None
|
||||
has_metadata: bool = False
|
||||
migration_after: tuple[str, ...] = ()
|
||||
migration_before: tuple[str, ...] = ()
|
||||
migration_tasks: tuple[ModuleMigrationTaskMetadata, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskMetadata:
|
||||
task_id: str
|
||||
phase: str
|
||||
summary: str
|
||||
task_version: str = "1"
|
||||
safety: str = "automatic"
|
||||
idempotent: bool = True
|
||||
timeout_seconds: int | None = None
|
||||
has_executor: bool = False
|
||||
|
||||
|
||||
def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan:
|
||||
@@ -28,4 +51,28 @@ def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Itera
|
||||
metadata.append(spec.metadata)
|
||||
if spec.script_location:
|
||||
script_locations.append(spec.script_location)
|
||||
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations))
|
||||
modules = tuple(
|
||||
ModuleMigrationMetadata(
|
||||
module_id=manifest.id,
|
||||
script_location=manifest.migration_spec.script_location,
|
||||
has_metadata=isinstance(manifest.migration_spec.metadata, MetaData),
|
||||
migration_after=tuple(manifest.migration_spec.migration_after),
|
||||
migration_before=tuple(manifest.migration_spec.migration_before),
|
||||
migration_tasks=tuple(
|
||||
ModuleMigrationTaskMetadata(
|
||||
task_id=task.task_id,
|
||||
phase=task.phase,
|
||||
summary=task.summary,
|
||||
task_version=task.task_version,
|
||||
safety=task.safety,
|
||||
idempotent=task.idempotent,
|
||||
timeout_seconds=task.timeout_seconds,
|
||||
has_executor=task.executor is not None,
|
||||
)
|
||||
for task in manifest.migration_spec.migration_tasks
|
||||
),
|
||||
)
|
||||
for manifest in registry.manifests()
|
||||
if manifest.migration_spec is not None
|
||||
)
|
||||
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations), modules=modules)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ MODULE_SETTINGS_KEY = "module_management"
|
||||
INSTALL_PLAN_KEY = "install_plan"
|
||||
REQUIRED_PLATFORM_MODULES = ("access",)
|
||||
PROTECTED_MODULES = (*REQUIRED_PLATFORM_MODULES, "admin")
|
||||
INSTALL_PLAN_ACTIONS = ("install", "uninstall")
|
||||
INSTALL_PLAN_ACTIONS = ("install", "update", "uninstall")
|
||||
INSTALL_PLAN_STATUSES = ("planned", "applied", "blocked")
|
||||
INSTALL_PLAN_SOURCES = ("manual", "catalog")
|
||||
LOCAL_DEPENDENCY_REF_PREFIXES = ("file:", "path:", "workspace:", "link:")
|
||||
@@ -46,6 +46,7 @@ class ModuleInstallPlanItem:
|
||||
webui_package: str | None = None
|
||||
webui_ref: str | None = None
|
||||
artifact_integrity: Mapping[str, object] | None = None
|
||||
data_safety_acknowledged: bool = False
|
||||
destroy_data: bool = False
|
||||
status: str = "planned"
|
||||
notes: str | None = None
|
||||
@@ -59,6 +60,8 @@ class ModuleInstallPlanItem:
|
||||
}
|
||||
if self.destroy_data:
|
||||
payload["destroy_data"] = True
|
||||
if self.data_safety_acknowledged:
|
||||
payload["data_safety_acknowledged"] = True
|
||||
if self.catalog:
|
||||
payload["catalog"] = dict(self.catalog)
|
||||
for key in ("python_package", "python_ref", "webui_package", "webui_ref", "notes"):
|
||||
@@ -206,7 +209,7 @@ def module_install_plan_commands(
|
||||
for item in items:
|
||||
if item.status not in included_statuses:
|
||||
continue
|
||||
if item.action == "install":
|
||||
if item.action in {"install", "update"}:
|
||||
if item.python_ref:
|
||||
commands.append(f"python -m pip install {shlex.quote(item.python_ref)}")
|
||||
if item.webui_package and item.webui_ref:
|
||||
@@ -281,7 +284,7 @@ def desired_modules_after_package_plan(
|
||||
removed = {item.module_id for item in planned_items if item.action == "uninstall"}
|
||||
desired = [module_id for module_id in desired if module_id not in removed]
|
||||
if activate_installed_modules:
|
||||
desired.extend(item.module_id for item in planned_items if item.action == "install")
|
||||
desired.extend(item.module_id for item in planned_items if item.action in {"install", "update"})
|
||||
return tuple(dict.fromkeys(desired))
|
||||
|
||||
|
||||
@@ -305,6 +308,7 @@ def normalize_module_install_plan_item(
|
||||
webui_package = _clean_optional_string(raw.get("webui_package"))
|
||||
webui_ref = _clean_optional_string(raw.get("webui_ref"))
|
||||
artifact_integrity = _clean_optional_mapping(raw.get("artifact_integrity"), field="artifact_integrity", module_id=module_id)
|
||||
data_safety_acknowledged = _clean_bool(raw.get("data_safety_acknowledged"))
|
||||
destroy_data = _clean_bool(raw.get("destroy_data"))
|
||||
notes = _clean_optional_string(raw.get("notes"))
|
||||
|
||||
@@ -314,13 +318,13 @@ def normalize_module_install_plan_item(
|
||||
raise ModuleManagementError(f"Unsupported install plan status for {module_id!r}: {status!r}.")
|
||||
if source not in INSTALL_PLAN_SOURCES:
|
||||
raise ModuleManagementError(f"Unsupported install plan source for {module_id!r}: {source!r}.")
|
||||
if action == "install" and not python_ref:
|
||||
if action in {"install", "update"} and not python_ref:
|
||||
raise ModuleManagementError(f"Install plan item {module_id!r} needs a Python package reference.")
|
||||
if action == "uninstall" and not python_package:
|
||||
raise ModuleManagementError(f"Uninstall plan item {module_id!r} needs a Python package name.")
|
||||
if action != "uninstall" and destroy_data:
|
||||
raise ModuleManagementError(f"Install plan item {module_id!r} can only destroy data during uninstall.")
|
||||
if action == "install" and bool(webui_package) != bool(webui_ref):
|
||||
if action in {"install", "update"} and bool(webui_package) != bool(webui_ref):
|
||||
raise ModuleManagementError(f"Install plan item {module_id!r} needs both WebUI package and WebUI reference, or neither.")
|
||||
if action == "uninstall" and webui_ref and not webui_package:
|
||||
raise ModuleManagementError(f"Uninstall plan item {module_id!r} has a WebUI reference but no WebUI package.")
|
||||
@@ -339,6 +343,7 @@ def normalize_module_install_plan_item(
|
||||
webui_package=webui_package,
|
||||
webui_ref=webui_ref,
|
||||
artifact_integrity=artifact_integrity,
|
||||
data_safety_acknowledged=data_safety_acknowledged,
|
||||
destroy_data=destroy_data,
|
||||
status=status,
|
||||
notes=notes,
|
||||
|
||||
@@ -19,6 +19,13 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey,
|
||||
from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range
|
||||
|
||||
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
||||
CATALOG_MIGRATION_SAFETY = ("automatic", "requires_review", "forward_only", "destructive")
|
||||
CATALOG_MIGRATION_TASK_PHASES = (
|
||||
"pre_migration_check",
|
||||
"pre_migration_prepare",
|
||||
"post_migration_backfill",
|
||||
"post_migration_verify",
|
||||
)
|
||||
|
||||
|
||||
def module_package_catalog(
|
||||
@@ -598,7 +605,7 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
||||
raise ValueError("Module package catalog entries must be objects.")
|
||||
module_id = _required_str(value, "module_id")
|
||||
action = str(value.get("action") or "install")
|
||||
if action not in {"install", "uninstall"}:
|
||||
if action not in {"install", "update", "uninstall"}:
|
||||
raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}")
|
||||
item = {
|
||||
"module_id": module_id,
|
||||
@@ -606,6 +613,21 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
||||
"description": _optional_str(value, "description"),
|
||||
"version": _optional_str(value, "version"),
|
||||
"action": action,
|
||||
"dependencies": _string_list(value.get("dependencies")),
|
||||
"optional_dependencies": _string_list(value.get("optional_dependencies")),
|
||||
"migration_safety": _catalog_migration_safety(value.get("migration_safety"), module_id=module_id),
|
||||
"migration_notes": _optional_str(value, "migration_notes"),
|
||||
"migration_after": _string_list(value.get("migration_after")),
|
||||
"migration_before": _string_list(value.get("migration_before")),
|
||||
"migration_tasks": _normalize_catalog_migration_tasks(value.get("migration_tasks"), module_id=module_id),
|
||||
"current_version_min": _optional_str(value, "current_version_min"),
|
||||
"current_version_max_exclusive": _optional_str(value, "current_version_max_exclusive"),
|
||||
"bridge_release": _optional_bool(value, "bridge_release"),
|
||||
"bridge_notes": _optional_str(value, "bridge_notes"),
|
||||
"allow_downgrade": _optional_bool(value, "allow_downgrade"),
|
||||
"allow_same_version": _optional_bool(value, "allow_same_version"),
|
||||
"recovery_tested": _optional_bool(value, "recovery_tested"),
|
||||
"recovery_notes": _optional_str(value, "recovery_notes"),
|
||||
"python_package": _optional_str(value, "python_package"),
|
||||
"python_ref": _optional_str(value, "python_ref"),
|
||||
"webui_package": _optional_str(value, "webui_package"),
|
||||
@@ -616,12 +638,91 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
||||
"notes": _optional_str(value, "notes"),
|
||||
"tags": _string_list(value.get("tags")),
|
||||
}
|
||||
if not version_range_is_valid(
|
||||
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||
):
|
||||
version_range = format_version_range(
|
||||
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||
)
|
||||
raise ValueError(f"Module package catalog entry {module_id!r} has invalid current-version range {version_range!r}.")
|
||||
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
|
||||
if artifact_integrity:
|
||||
item["artifact_integrity"] = artifact_integrity
|
||||
return item
|
||||
|
||||
|
||||
def _catalog_migration_safety(value: Any, *, module_id: str) -> str:
|
||||
if value is None:
|
||||
return "automatic"
|
||||
safety = str(value).strip()
|
||||
if safety not in CATALOG_MIGRATION_SAFETY:
|
||||
allowed = ", ".join(CATALOG_MIGRATION_SAFETY)
|
||||
raise ValueError(f"Unsupported migration_safety for {module_id!r}: {safety!r}; expected one of {allowed}.")
|
||||
return safety
|
||||
|
||||
|
||||
def _normalize_catalog_migration_tasks(value: Any, *, module_id: str) -> list[dict[str, object]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise ValueError(f"Module package catalog migration_tasks for {module_id!r} must be a list.")
|
||||
normalized: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
for raw in value:
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"Module package catalog migration_tasks entries for {module_id!r} must be objects.")
|
||||
task_id = _catalog_task_required_str(raw, "task_id", module_id=module_id)
|
||||
if task_id in seen:
|
||||
raise ValueError(f"Module package catalog entry {module_id!r} declares migration task {task_id!r} more than once.")
|
||||
seen.add(task_id)
|
||||
phase = _catalog_task_required_str(raw, "phase", module_id=module_id)
|
||||
if phase not in CATALOG_MIGRATION_TASK_PHASES:
|
||||
allowed = ", ".join(CATALOG_MIGRATION_TASK_PHASES)
|
||||
raise ValueError(f"Unsupported migration task phase for {module_id!r}/{task_id!r}: {phase!r}; expected one of {allowed}.")
|
||||
summary = _catalog_task_required_str(raw, "summary", module_id=module_id)
|
||||
safety = _catalog_migration_safety(raw.get("safety"), module_id=module_id)
|
||||
item: dict[str, object] = {
|
||||
"task_id": task_id,
|
||||
"phase": phase,
|
||||
"summary": summary,
|
||||
"task_version": _optional_str(raw, "task_version") or "1",
|
||||
"safety": safety,
|
||||
"idempotent": _catalog_optional_bool_default(raw, "idempotent", default=True),
|
||||
}
|
||||
timeout_seconds = _catalog_optional_positive_int(raw, "timeout_seconds", module_id=module_id, task_id=task_id)
|
||||
if timeout_seconds is not None:
|
||||
item["timeout_seconds"] = timeout_seconds
|
||||
normalized.append(item)
|
||||
return normalized
|
||||
|
||||
|
||||
def _catalog_task_required_str(value: dict[str, Any], key: str, *, module_id: str) -> str:
|
||||
item = _optional_str(value, key)
|
||||
if not item:
|
||||
raise ValueError(f"Module package catalog migration task for {module_id!r} is missing {key!r}.")
|
||||
return item
|
||||
|
||||
|
||||
def _catalog_optional_bool_default(value: dict[str, Any], key: str, *, default: bool) -> bool:
|
||||
if key not in value:
|
||||
return default
|
||||
return _optional_bool(value, key)
|
||||
|
||||
|
||||
def _catalog_optional_positive_int(value: dict[str, Any], key: str, *, module_id: str, task_id: str) -> int | None:
|
||||
if value.get(key) is None:
|
||||
return None
|
||||
try:
|
||||
integer = int(value[key])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} has invalid {key!r}.") from exc
|
||||
if integer <= 0:
|
||||
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} requires positive {key!r}.")
|
||||
return integer
|
||||
|
||||
|
||||
def _required_str(value: dict[str, Any], key: str) -> str:
|
||||
item = _optional_str(value, key)
|
||||
if not item:
|
||||
|
||||
@@ -13,6 +13,14 @@ SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1"
|
||||
|
||||
PermissionLevel = Literal["system", "tenant"]
|
||||
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
||||
MigrationTaskPhase = Literal[
|
||||
"pre_migration_check",
|
||||
"pre_migration_prepare",
|
||||
"post_migration_backfill",
|
||||
"post_migration_verify",
|
||||
]
|
||||
MigrationTaskSafety = Literal["automatic", "requires_review", "forward_only", "destructive"]
|
||||
MigrationTaskStatus = Literal["ok", "warning", "blocked", "skipped"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -91,11 +99,49 @@ MigrationRetirementExecutor = Callable[[object, str], None]
|
||||
MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskContext:
|
||||
module_id: str
|
||||
task_id: str
|
||||
phase: MigrationTaskPhase
|
||||
database_url: str
|
||||
target_version: str | None = None
|
||||
session: object | None = None
|
||||
dry_run: bool = False
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTaskResult:
|
||||
status: MigrationTaskStatus = "ok"
|
||||
message: str | None = None
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
ModuleMigrationTaskExecutor = Callable[[ModuleMigrationTaskContext], ModuleMigrationTaskResult | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleMigrationTask:
|
||||
task_id: str
|
||||
phase: MigrationTaskPhase
|
||||
summary: str
|
||||
task_version: str = "1"
|
||||
safety: MigrationTaskSafety = "automatic"
|
||||
idempotent: bool = True
|
||||
timeout_seconds: int | None = None
|
||||
executor: ModuleMigrationTaskExecutor | None = None
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationSpec:
|
||||
module_id: str
|
||||
metadata: object | None = None
|
||||
script_location: str | None = None
|
||||
migration_after: tuple[str, ...] = ()
|
||||
migration_before: tuple[str, ...] = ()
|
||||
migration_tasks: tuple[ModuleMigrationTask, ...] = ()
|
||||
retirement_supported: bool = False
|
||||
retirement_provider: MigrationRetirementProvider | None = None
|
||||
retirement_notes: str | None = None
|
||||
@@ -215,7 +261,28 @@ class ResourceAclProvider(Protocol):
|
||||
|
||||
|
||||
TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
|
||||
DeleteVetoProvider = Callable[[object, str, str], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeleteVetoIssue:
|
||||
severity: Literal["blocker", "warning", "info"]
|
||||
code: str
|
||||
message: str
|
||||
module_id: str | None = None
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
DeleteVetoProviderResult = DeleteVetoIssue | Iterable[DeleteVetoIssue] | None
|
||||
DeleteVetoProvider = Callable[[object, str, str], DeleteVetoProviderResult]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeleteVetoProviderRegistration:
|
||||
module_id: str
|
||||
resource_type: str
|
||||
provider: DeleteVetoProvider
|
||||
|
||||
|
||||
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
||||
CapabilityFactory = Callable[[ModuleContext], object]
|
||||
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]
|
||||
|
||||
@@ -3,11 +3,13 @@ from __future__ import annotations
|
||||
import re
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from govoplan_core.core.modules import (
|
||||
CapabilityFactory,
|
||||
DeleteVetoIssue,
|
||||
DeleteVetoProvider,
|
||||
DeleteVetoProviderRegistration,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleInterfaceRequirement,
|
||||
ModuleContext,
|
||||
@@ -45,7 +47,7 @@ class PlatformRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._manifests: dict[str, ModuleManifest] = {}
|
||||
self._tenant_summary_providers: dict[str, TenantSummaryProvider] = {}
|
||||
self._delete_veto_providers: dict[str, list[DeleteVetoProvider]] = defaultdict(list)
|
||||
self._delete_veto_providers: dict[str, list[DeleteVetoProviderRegistration]] = defaultdict(list)
|
||||
self._capability_factories: dict[str, CapabilityFactory] = {}
|
||||
self._capabilities: dict[str, object] = {}
|
||||
self._capability_context: ModuleContext | None = None
|
||||
@@ -58,7 +60,7 @@ class PlatformRegistry:
|
||||
self.register_tenant_summary_provider(manifest.id, provider)
|
||||
for resource_type, providers in manifest.delete_veto_providers.items():
|
||||
for provider in providers:
|
||||
self.register_delete_veto(resource_type, provider)
|
||||
self.register_delete_veto(manifest.id, resource_type, provider)
|
||||
for name, factory in manifest.capability_factories.items():
|
||||
self.register_capability_factory(manifest.id, name, factory)
|
||||
return manifest
|
||||
@@ -148,12 +150,39 @@ class PlatformRegistry:
|
||||
def tenant_summary_providers(self) -> Mapping[str, TenantSummaryProvider]:
|
||||
return dict(self._tenant_summary_providers)
|
||||
|
||||
def register_delete_veto(self, resource_type: str, provider: DeleteVetoProvider) -> None:
|
||||
self._delete_veto_providers[resource_type].append(provider)
|
||||
def register_delete_veto(self, module_id: str, resource_type: str, provider: DeleteVetoProvider) -> None:
|
||||
self._delete_veto_providers[resource_type].append(DeleteVetoProviderRegistration(
|
||||
module_id=module_id,
|
||||
resource_type=resource_type,
|
||||
provider=provider,
|
||||
))
|
||||
|
||||
def delete_veto_providers(self, resource_type: str) -> tuple[DeleteVetoProvider, ...]:
|
||||
return tuple(registration.provider for registration in self._delete_veto_providers.get(resource_type, ()))
|
||||
|
||||
def delete_veto_provider_registrations(self, resource_type: str) -> tuple[DeleteVetoProviderRegistration, ...]:
|
||||
return tuple(self._delete_veto_providers.get(resource_type, ()))
|
||||
|
||||
def collect_delete_veto_issues(self, resource_type: str, session: object, tenant_id: str, resource_id: str) -> tuple[DeleteVetoIssue, ...]:
|
||||
issues: list[DeleteVetoIssue] = []
|
||||
for registration in self.delete_veto_provider_registrations(resource_type):
|
||||
try:
|
||||
result = registration.provider(session, tenant_id, resource_id)
|
||||
except Exception as exc:
|
||||
issues.append(DeleteVetoIssue(
|
||||
severity="blocker",
|
||||
code="module_veto",
|
||||
message=str(exc),
|
||||
module_id=registration.module_id,
|
||||
details={
|
||||
"resource_type": resource_type,
|
||||
"resource_id": resource_id,
|
||||
},
|
||||
))
|
||||
continue
|
||||
issues.extend(_normalize_delete_veto_result(result, registration=registration, resource_id=resource_id))
|
||||
return tuple(issues)
|
||||
|
||||
def validate(self) -> RegistrySnapshot:
|
||||
ordered = tuple(self._topologically_sorted())
|
||||
available_capabilities = set(self._capability_factories)
|
||||
@@ -224,6 +253,52 @@ class PlatformRegistry:
|
||||
return (self._manifests[module_id] for module_id in ordered)
|
||||
|
||||
|
||||
def _normalize_delete_veto_result(
|
||||
result: object,
|
||||
*,
|
||||
registration: DeleteVetoProviderRegistration,
|
||||
resource_id: str,
|
||||
) -> tuple[DeleteVetoIssue, ...]:
|
||||
if result is None:
|
||||
return ()
|
||||
if isinstance(result, DeleteVetoIssue):
|
||||
return (_attribute_delete_veto_issue(result, registration=registration, resource_id=resource_id),)
|
||||
if isinstance(result, str):
|
||||
return (DeleteVetoIssue(
|
||||
severity="blocker",
|
||||
code="module_veto",
|
||||
message=result,
|
||||
module_id=registration.module_id,
|
||||
details={
|
||||
"resource_type": registration.resource_type,
|
||||
"resource_id": resource_id,
|
||||
},
|
||||
),)
|
||||
if isinstance(result, Iterable):
|
||||
return tuple(
|
||||
_attribute_delete_veto_issue(issue, registration=registration, resource_id=resource_id)
|
||||
for issue in result
|
||||
if isinstance(issue, DeleteVetoIssue)
|
||||
)
|
||||
return ()
|
||||
|
||||
|
||||
def _attribute_delete_veto_issue(
|
||||
issue: DeleteVetoIssue,
|
||||
*,
|
||||
registration: DeleteVetoProviderRegistration,
|
||||
resource_id: str,
|
||||
) -> DeleteVetoIssue:
|
||||
details = {
|
||||
"resource_type": registration.resource_type,
|
||||
"resource_id": resource_id,
|
||||
**dict(issue.details),
|
||||
}
|
||||
if issue.module_id == registration.module_id and issue.details == details:
|
||||
return issue
|
||||
return replace(issue, module_id=issue.module_id or registration.module_id, details=details)
|
||||
|
||||
|
||||
def _validate_manifest_shape(manifest: ModuleManifest) -> None:
|
||||
if not _MODULE_ID_RE.match(manifest.id):
|
||||
raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}")
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan
|
||||
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||
from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor
|
||||
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||
from govoplan_core.db.session import configure_database
|
||||
from govoplan_core.core.modules import ModuleMigrationTaskContext, ModuleMigrationTaskResult
|
||||
from govoplan_core.db.session import configure_database, get_database
|
||||
from govoplan_core.server.default_config import get_server_config
|
||||
from govoplan_core.server.config import ManifestFactory
|
||||
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||
@@ -33,6 +38,12 @@ REVISION_CORE_CHANGE_SEQUENCE = "3f4a5b6c7d8e"
|
||||
REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0"
|
||||
LEGACY_SCOPE_TABLE = "tenancy_tenants"
|
||||
CORE_SCOPE_TABLE = "core_scopes"
|
||||
PRE_MIGRATION_TASK_PHASES = ("pre_migration_check", "pre_migration_prepare")
|
||||
POST_MIGRATION_TASK_PHASES = ("post_migration_backfill", "post_migration_verify")
|
||||
MIGRATION_TASK_PHASES = (*PRE_MIGRATION_TASK_PHASES, *POST_MIGRATION_TASK_PHASES)
|
||||
MIGRATION_TRACK_RELEASE = "release"
|
||||
MIGRATION_TRACK_DEV = "dev"
|
||||
MIGRATION_TRACKS = (MIGRATION_TRACK_RELEASE, MIGRATION_TRACK_DEV)
|
||||
|
||||
_NAMESPACE_TABLE_RENAMES = (
|
||||
("tenants", "tenancy_tenants"),
|
||||
@@ -160,12 +171,33 @@ class MigrationResult:
|
||||
current_revision: str | None
|
||||
|
||||
|
||||
class ModuleMigrationTaskExecutionError(RuntimeError):
|
||||
def __init__(self, message: str, *, records: tuple[dict[str, object], ...]) -> None:
|
||||
super().__init__(message)
|
||||
self.records = records
|
||||
|
||||
|
||||
def registered_module_migration_plan(
|
||||
database_url: str | None = None,
|
||||
*,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
migration_track: str | None = None,
|
||||
) -> MigrationMetadataPlan:
|
||||
plan = migration_metadata_plan(_registered_module_registry(
|
||||
database_url=database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
))
|
||||
return _migration_plan_for_track(plan, _normalize_migration_track(migration_track))
|
||||
|
||||
|
||||
def _registered_module_registry(
|
||||
*,
|
||||
database_url: str | None = None,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
):
|
||||
server_config = get_server_config()
|
||||
active_database_url = database_url or settings.database_url
|
||||
if active_database_url:
|
||||
@@ -187,11 +219,208 @@ def registered_module_migration_plan(
|
||||
active_enabled_modules,
|
||||
manifest_factories=active_manifest_factories,
|
||||
)
|
||||
return migration_metadata_plan(registry)
|
||||
return registry
|
||||
|
||||
|
||||
def run_registered_module_migration_tasks(
|
||||
*,
|
||||
database_url: str | None = None,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
migration_module_order: tuple[str, ...] | list[str] | None = None,
|
||||
phases: tuple[str, ...] | list[str] = MIGRATION_TASK_PHASES,
|
||||
dry_run: bool = False,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
active_phases = tuple(dict.fromkeys(str(item).strip() for item in phases if str(item).strip()))
|
||||
invalid_phases = tuple(phase for phase in active_phases if phase not in MIGRATION_TASK_PHASES)
|
||||
if invalid_phases:
|
||||
raise ValueError("Unsupported module migration task phase(s): " + ", ".join(invalid_phases))
|
||||
url = database_url or settings.database_url
|
||||
registry = _registered_module_registry(
|
||||
database_url=url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
)
|
||||
manifests = {manifest.id: manifest for manifest in registry.manifests()}
|
||||
ordered_ids = tuple(dict.fromkeys([
|
||||
*(str(item).strip() for item in (migration_module_order or ()) if str(item).strip()),
|
||||
*manifests.keys(),
|
||||
]))
|
||||
records: list[dict[str, object]] = []
|
||||
database = get_database()
|
||||
with database.SessionLocal() as session:
|
||||
for phase in active_phases:
|
||||
for module_id in ordered_ids:
|
||||
manifest = manifests.get(module_id)
|
||||
if manifest is None or manifest.migration_spec is None:
|
||||
continue
|
||||
for task in manifest.migration_spec.migration_tasks:
|
||||
if task.phase != phase:
|
||||
continue
|
||||
record: dict[str, object] = {
|
||||
"module_id": manifest.id,
|
||||
"task_id": task.task_id,
|
||||
"phase": task.phase,
|
||||
"summary": task.summary,
|
||||
"task_version": task.task_version,
|
||||
"safety": task.safety,
|
||||
"idempotent": task.idempotent,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
if task.timeout_seconds is not None:
|
||||
record["timeout_seconds"] = task.timeout_seconds
|
||||
if not task.idempotent:
|
||||
record.update({"status": "blocked", "message": "Task is not idempotent."})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} is not idempotent.",
|
||||
records=tuple(records),
|
||||
)
|
||||
if dry_run:
|
||||
record.update({"status": "skipped", "message": "Dry run; executor was not called."})
|
||||
records.append(record)
|
||||
continue
|
||||
if task.executor is None:
|
||||
record.update({"status": "blocked", "message": "Task has no executor."})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} has no executor.",
|
||||
records=tuple(records),
|
||||
)
|
||||
context = ModuleMigrationTaskContext(
|
||||
module_id=manifest.id,
|
||||
task_id=task.task_id,
|
||||
phase=task.phase,
|
||||
database_url=url,
|
||||
target_version=manifest.version,
|
||||
session=session,
|
||||
dry_run=dry_run,
|
||||
metadata=task.metadata,
|
||||
)
|
||||
try:
|
||||
result = task.executor(context)
|
||||
normalized = _normalize_migration_task_result(result)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
record.update({
|
||||
"status": "blocked",
|
||||
"message": f"{type(exc).__name__}: {exc}",
|
||||
})
|
||||
records.append(record)
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
f"Module migration task {manifest.id}/{task.task_id} failed.",
|
||||
records=tuple(records),
|
||||
) from exc
|
||||
record.update({
|
||||
"status": normalized.status,
|
||||
"message": normalized.message,
|
||||
"details": _jsonable_migration_task_details(normalized.details),
|
||||
})
|
||||
records.append(record)
|
||||
if normalized.status == "blocked":
|
||||
session.rollback()
|
||||
raise ModuleMigrationTaskExecutionError(
|
||||
normalized.message or f"Module migration task {manifest.id}/{task.task_id} blocked migration.",
|
||||
records=tuple(records),
|
||||
)
|
||||
session.commit()
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _normalize_migration_task_result(result: ModuleMigrationTaskResult | None) -> ModuleMigrationTaskResult:
|
||||
if result is None:
|
||||
return ModuleMigrationTaskResult()
|
||||
if not isinstance(result, ModuleMigrationTaskResult):
|
||||
raise TypeError("Module migration task executors must return ModuleMigrationTaskResult or None.")
|
||||
if result.status not in {"ok", "warning", "blocked", "skipped"}:
|
||||
raise ValueError(f"Unsupported module migration task status: {result.status!r}")
|
||||
return result
|
||||
|
||||
|
||||
def _jsonable_migration_task_details(value: Mapping[str, Any]) -> Mapping[str, Any] | str:
|
||||
try:
|
||||
json.dumps(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(dict(value))
|
||||
return value
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
packaged_root = Path(__file__).resolve().parents[3]
|
||||
configured = os.environ.get("GOVOPLAN_CORE_SOURCE_ROOT")
|
||||
candidates = [
|
||||
Path(configured).expanduser() if configured else None,
|
||||
Path.cwd(),
|
||||
packaged_root,
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate is None:
|
||||
continue
|
||||
resolved = candidate.resolve()
|
||||
if (resolved / "alembic.ini").exists() and (resolved / "alembic").is_dir():
|
||||
return resolved
|
||||
return packaged_root
|
||||
|
||||
|
||||
def _normalize_migration_track(value: str | None = None) -> str:
|
||||
raw_value = value if value is not None else settings.migration_track
|
||||
track = str(raw_value or MIGRATION_TRACK_RELEASE).strip().lower()
|
||||
aliases = {
|
||||
"baseline": MIGRATION_TRACK_RELEASE,
|
||||
"prod": MIGRATION_TRACK_RELEASE,
|
||||
"production": MIGRATION_TRACK_RELEASE,
|
||||
"releases": MIGRATION_TRACK_RELEASE,
|
||||
"detailed": MIGRATION_TRACK_DEV,
|
||||
"development": MIGRATION_TRACK_DEV,
|
||||
}
|
||||
track = aliases.get(track, track)
|
||||
if track not in MIGRATION_TRACKS:
|
||||
raise ValueError(
|
||||
"Unsupported migration track "
|
||||
f"{raw_value!r}; expected one of: {', '.join(MIGRATION_TRACKS)}"
|
||||
)
|
||||
return track
|
||||
|
||||
|
||||
def _core_version_location(root: Path, migration_track: str) -> Path:
|
||||
if migration_track == MIGRATION_TRACK_DEV:
|
||||
dev_versions = root / "alembic" / "dev_versions"
|
||||
if dev_versions.is_dir():
|
||||
return dev_versions
|
||||
return root / "alembic" / "versions"
|
||||
|
||||
|
||||
def _script_location_for_track(location: str | None, migration_track: str) -> str | None:
|
||||
if not location or migration_track != MIGRATION_TRACK_DEV:
|
||||
return location
|
||||
path = Path(location)
|
||||
if path.name != "versions":
|
||||
return location
|
||||
dev_location = path.with_name("dev_versions")
|
||||
if dev_location.is_dir():
|
||||
return str(dev_location)
|
||||
return location
|
||||
|
||||
|
||||
def _migration_plan_for_track(plan: MigrationMetadataPlan, migration_track: str) -> MigrationMetadataPlan:
|
||||
if migration_track == MIGRATION_TRACK_RELEASE:
|
||||
return plan
|
||||
script_locations = tuple(
|
||||
location
|
||||
for location in (
|
||||
_script_location_for_track(location, migration_track)
|
||||
for location in plan.script_locations
|
||||
)
|
||||
if location
|
||||
)
|
||||
modules = tuple(
|
||||
replace(
|
||||
item,
|
||||
script_location=_script_location_for_track(item.script_location, migration_track),
|
||||
)
|
||||
for item in plan.modules
|
||||
)
|
||||
return replace(plan, script_locations=script_locations, modules=modules)
|
||||
|
||||
|
||||
def alembic_config(
|
||||
@@ -199,8 +428,10 @@ def alembic_config(
|
||||
database_url: str | None = None,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
migration_track: str | None = None,
|
||||
) -> Config:
|
||||
root = _repo_root()
|
||||
active_migration_track = _normalize_migration_track(migration_track)
|
||||
config = Config(str(root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(root / "alembic"))
|
||||
|
||||
@@ -209,13 +440,15 @@ def alembic_config(
|
||||
active_database_url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
migration_track=active_migration_track,
|
||||
)
|
||||
version_locations = [str(root / "alembic" / "versions")]
|
||||
version_locations = [str(_core_version_location(root, active_migration_track))]
|
||||
version_locations.extend(location for location in plan.script_locations if location)
|
||||
config.set_main_option("version_locations", os.pathsep.join(dict.fromkeys(version_locations)))
|
||||
config.set_main_option("version_path_separator", "os")
|
||||
config.set_main_option("path_separator", "os")
|
||||
|
||||
config.attributes["database_url"] = active_database_url
|
||||
config.attributes["migration_track"] = active_migration_track
|
||||
if enabled_modules is not None:
|
||||
config.attributes["enabled_modules"] = tuple(enabled_modules)
|
||||
if manifest_factories:
|
||||
@@ -439,7 +672,67 @@ def reconcile_change_sequence_retention_floor_drift(database_url: str | None = N
|
||||
return changed
|
||||
|
||||
|
||||
def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None:
|
||||
def reconcile_covered_alembic_dependency_heads(
|
||||
database_url: str | None = None,
|
||||
*,
|
||||
config: Config | None = None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Remove stale parent/dependency rows from ``alembic_version``.
|
||||
|
||||
Alembic's version table represents current graph heads, not every owner
|
||||
head that contributed to the schema. After the v0.1.7 baseline squash,
|
||||
development databases may still carry dependency parents such as the core
|
||||
baseline beside child module heads. That shape is semantically already
|
||||
upgraded, but Alembic rejects it as an overlapping current revision set.
|
||||
"""
|
||||
|
||||
url = database_url or settings.database_url
|
||||
active_config = config or alembic_config(database_url=url)
|
||||
script = ScriptDirectory.from_config(active_config)
|
||||
engine = create_engine(url)
|
||||
removed: set[str] = set()
|
||||
try:
|
||||
with engine.begin() as connection:
|
||||
tables = set(inspect(connection).get_table_names())
|
||||
if "alembic_version" not in tables:
|
||||
return ()
|
||||
current = {
|
||||
str(row[0])
|
||||
for row in connection.execute(text("SELECT version_num FROM alembic_version")).all()
|
||||
if row[0]
|
||||
}
|
||||
if len(current) < 2:
|
||||
return ()
|
||||
covered: set[str] = set()
|
||||
for revision_id in current:
|
||||
try:
|
||||
revision = script.get_revision(revision_id)
|
||||
except Exception:
|
||||
continue
|
||||
ancestors = {
|
||||
item.revision
|
||||
for item in script.revision_map._get_ancestor_nodes( # noqa: SLF001 - Alembic has no public dependency-ancestor API.
|
||||
[revision],
|
||||
include_dependencies=True,
|
||||
)
|
||||
}
|
||||
covered.update(ancestors - {revision_id})
|
||||
removed = current.intersection(covered)
|
||||
for revision_id in sorted(removed):
|
||||
connection.execute(
|
||||
text("DELETE FROM alembic_version WHERE version_num = :revision"),
|
||||
{"revision": revision_id},
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
return tuple(sorted(removed))
|
||||
|
||||
|
||||
def reconcile_legacy_create_all_schema(
|
||||
database_url: str | None = None,
|
||||
*,
|
||||
migration_track: str | None = None,
|
||||
) -> str | None:
|
||||
"""Repair the known Alembic/create_all drift without modifying application data.
|
||||
|
||||
Returns the revision stamped during reconciliation, or ``None`` when no
|
||||
@@ -496,7 +789,7 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str |
|
||||
if target is None:
|
||||
return None
|
||||
|
||||
command.stamp(alembic_config(database_url=url), target)
|
||||
command.stamp(alembic_config(database_url=url, migration_track=migration_track), target)
|
||||
return target
|
||||
|
||||
|
||||
@@ -505,22 +798,37 @@ def migrate_database(
|
||||
database_url: str | None = None,
|
||||
reconcile_legacy_schema: bool = True,
|
||||
enabled_modules: tuple[str, ...] | list[str] | None = None,
|
||||
migration_module_order: tuple[str, ...] | list[str] | None = None,
|
||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||
migration_track: str | None = None,
|
||||
) -> MigrationResult:
|
||||
url = database_url or settings.database_url
|
||||
active_migration_track = _normalize_migration_track(migration_track)
|
||||
if reconcile_legacy_schema:
|
||||
reconcile_namespace_table_drift(url)
|
||||
reconcile_change_sequence_retention_floor_drift(url)
|
||||
previous = database_revision(url)
|
||||
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
|
||||
command.upgrade(
|
||||
alembic_config(
|
||||
database_url=url,
|
||||
reconciled = (
|
||||
reconcile_legacy_create_all_schema(url, migration_track=active_migration_track)
|
||||
if reconcile_legacy_schema
|
||||
else None
|
||||
)
|
||||
config = alembic_config(
|
||||
database_url=url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
migration_track=active_migration_track,
|
||||
)
|
||||
reconcile_covered_alembic_dependency_heads(url, config=config)
|
||||
if migration_module_order:
|
||||
plan = registered_module_migration_plan(
|
||||
url,
|
||||
enabled_modules=enabled_modules,
|
||||
manifest_factories=manifest_factories,
|
||||
),
|
||||
"heads",
|
||||
)
|
||||
migration_track=active_migration_track,
|
||||
)
|
||||
_upgrade_ordered_module_heads(config, plan, tuple(migration_module_order))
|
||||
command.upgrade(config, "heads")
|
||||
if reconcile_legacy_schema:
|
||||
reconcile_scope_table_retirement_drift(url)
|
||||
current = database_revision(url)
|
||||
@@ -529,3 +837,43 @@ def migrate_database(
|
||||
reconciled_revision=reconciled,
|
||||
current_revision=current,
|
||||
)
|
||||
|
||||
|
||||
def _upgrade_ordered_module_heads(
|
||||
config: Config,
|
||||
plan: MigrationMetadataPlan,
|
||||
module_order: tuple[str, ...],
|
||||
) -> None:
|
||||
module_heads = _module_heads_by_id(config, plan)
|
||||
upgraded: set[str] = set()
|
||||
for module_id in dict.fromkeys(module_order):
|
||||
for head in module_heads.get(module_id, ()):
|
||||
if head in upgraded:
|
||||
continue
|
||||
command.upgrade(config, head)
|
||||
upgraded.add(head)
|
||||
|
||||
|
||||
def _module_heads_by_id(config: Config, plan: MigrationMetadataPlan) -> dict[str, tuple[str, ...]]:
|
||||
script = ScriptDirectory.from_config(config)
|
||||
heads = script.get_heads()
|
||||
module_locations = {
|
||||
item.module_id: Path(item.script_location).resolve()
|
||||
for item in plan.modules
|
||||
if item.script_location
|
||||
}
|
||||
grouped: dict[str, list[str]] = {module_id: [] for module_id in module_locations}
|
||||
for head in heads:
|
||||
revision = script.get_revision(head)
|
||||
path_text = getattr(revision, "path", None)
|
||||
if revision is None or not path_text:
|
||||
continue
|
||||
revision_path = Path(path_text).resolve()
|
||||
for module_id, location in module_locations.items():
|
||||
try:
|
||||
revision_path.relative_to(location)
|
||||
except ValueError:
|
||||
continue
|
||||
grouped[module_id].append(head)
|
||||
break
|
||||
return {module_id: tuple(sorted(values)) for module_id, values in grouped.items() if values}
|
||||
|
||||
@@ -22,7 +22,7 @@ DEFAULT_APP = "govoplan_core.server.app:app"
|
||||
DEFAULT_CONFIG = "govoplan_core.server.default_config:get_server_config"
|
||||
DEFAULT_DEV_DATABASE_URL = "postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev"
|
||||
DEFAULT_DEV_DATABASE_URL_PGTOOLS = "postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev"
|
||||
DEFAULT_SQLITE_DATABASE_NAME = "multimailer-dev.db"
|
||||
DEFAULT_SQLITE_DATABASE_NAME = "govoplan-dev.db"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -352,7 +352,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
try:
|
||||
import uvicorn
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit("uvicorn is not installed. Install govoplan-core with the server extra or requirements-dev.txt.") from exc
|
||||
raise SystemExit("uvicorn is not installed. Install govoplan-core with the server extra or the meta requirements-dev.txt.") from exc
|
||||
|
||||
uvicorn.run(
|
||||
args.app,
|
||||
|
||||
@@ -6,6 +6,42 @@ from govoplan_core.security.permissions import scopes_grant
|
||||
|
||||
|
||||
LEGACY_TO_MODULE_SCOPES: dict[str, str] = {
|
||||
"admin:users:read": "access:membership:read",
|
||||
"admin:users:create": "access:membership:create",
|
||||
"admin:users:update": "access:membership:update",
|
||||
"admin:users:suspend": "access:membership:update",
|
||||
"admin:groups:read": "access:group:read",
|
||||
"admin:groups:write": "access:group:write",
|
||||
"admin:groups:manage_members": "access:group:manage_members",
|
||||
"admin:roles:read": "access:role:read",
|
||||
"admin:roles:write": "access:role:write",
|
||||
"admin:roles:assign": "access:role:assign",
|
||||
"admin:api_keys:read": "access:api_key:read",
|
||||
"admin:api_keys:create": "access:api_key:create",
|
||||
"admin:api_keys:revoke": "access:api_key:revoke",
|
||||
"admin:settings:read": "access:setting:read",
|
||||
"admin:settings:write": "access:setting:write",
|
||||
"admin:policies:read": "access:policy:read",
|
||||
"admin:policies:write": "access:policy:write",
|
||||
"system:tenants:read": "access:tenant:read",
|
||||
"system:tenants:create": "access:tenant:create",
|
||||
"system:tenants:update": "access:tenant:update",
|
||||
"system:tenants:suspend": "access:tenant:suspend",
|
||||
"system:accounts:read": "access:account:read",
|
||||
"system:accounts:create": "access:account:create",
|
||||
"system:accounts:update": "access:account:update",
|
||||
"system:accounts:suspend": "access:account:suspend",
|
||||
"system:roles:read": "access:system_role:read",
|
||||
"system:roles:write": "access:system_role:write",
|
||||
"system:roles:assign": "access:system_role:assign",
|
||||
"system:access:read": "access:system_role:read",
|
||||
"system:access:assign": "access:system_role:assign",
|
||||
"system:audit:read": "access:audit:read",
|
||||
"system:settings:read": "access:system_setting:read",
|
||||
"system:settings:write": "access:system_setting:write",
|
||||
"system:maintenance:access": "access:maintenance:access",
|
||||
"system:governance:read": "access:governance:read",
|
||||
"system:governance:write": "access:governance:write",
|
||||
"campaign:read": "campaigns:campaign:read",
|
||||
"campaign:create": "campaigns:campaign:create",
|
||||
"campaign:update": "campaigns:campaign:update",
|
||||
@@ -44,14 +80,28 @@ LEGACY_TO_MODULE_SCOPES: dict[str, str] = {
|
||||
"mail_servers:manage_credentials": "mail:secret:manage",
|
||||
}
|
||||
|
||||
MODULE_TO_LEGACY_SCOPES = {module: legacy for legacy, module in LEGACY_TO_MODULE_SCOPES.items()}
|
||||
MODULE_TENANT_SCOPES = frozenset(LEGACY_TO_MODULE_SCOPES.values())
|
||||
MODULE_TO_LEGACY_SCOPE_ALIASES: dict[str, tuple[str, ...]] = {}
|
||||
for legacy_scope, module_scope in LEGACY_TO_MODULE_SCOPES.items():
|
||||
MODULE_TO_LEGACY_SCOPE_ALIASES[module_scope] = (
|
||||
*MODULE_TO_LEGACY_SCOPE_ALIASES.get(module_scope, ()),
|
||||
legacy_scope,
|
||||
)
|
||||
MODULE_TO_LEGACY_SCOPES = {
|
||||
module_scope: aliases[0]
|
||||
for module_scope, aliases in MODULE_TO_LEGACY_SCOPE_ALIASES.items()
|
||||
}
|
||||
MODULE_SYSTEM_SCOPES = frozenset(
|
||||
module
|
||||
for legacy, module in LEGACY_TO_MODULE_SCOPES.items()
|
||||
if legacy.startswith("system:")
|
||||
)
|
||||
MODULE_TENANT_SCOPES = frozenset(LEGACY_TO_MODULE_SCOPES.values()) - MODULE_SYSTEM_SCOPES
|
||||
|
||||
|
||||
def compatible_required_scopes(required: str) -> tuple[str, ...]:
|
||||
legacy = MODULE_TO_LEGACY_SCOPES.get(required)
|
||||
if legacy:
|
||||
return (required, legacy)
|
||||
legacy_aliases = MODULE_TO_LEGACY_SCOPE_ALIASES.get(required)
|
||||
if legacy_aliases:
|
||||
return (required, *legacy_aliases)
|
||||
module = LEGACY_TO_MODULE_SCOPES.get(required)
|
||||
if module:
|
||||
return (required, module)
|
||||
@@ -60,10 +110,18 @@ def compatible_required_scopes(required: str) -> tuple[str, ...]:
|
||||
|
||||
def scopes_grant_compatible(scopes: Iterable[str], required: str) -> bool:
|
||||
granted = list(scopes)
|
||||
if required in MODULE_SYSTEM_SCOPES:
|
||||
return "*" in granted or "system:*" in granted or any(
|
||||
scope != "tenant:*" and scopes_grant([scope], alias)
|
||||
for scope in granted
|
||||
for alias in compatible_required_scopes(required)
|
||||
)
|
||||
if scopes_grant(granted, required):
|
||||
return True
|
||||
if "tenant:*" in granted or "*" in granted:
|
||||
if required in MODULE_TENANT_SCOPES:
|
||||
return True
|
||||
if "system:*" in granted or "*" in granted:
|
||||
if required in MODULE_SYSTEM_SCOPES:
|
||||
return True
|
||||
return any(scopes_grant(granted, alias) for alias in compatible_required_scopes(required) if alias != required)
|
||||
|
||||
|
||||
47
src/govoplan_core/security/redaction.py
Normal file
47
src/govoplan_core/security/redaction.py
Normal file
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
def sensitive_key_tokens(key: object) -> set[str]:
|
||||
value = str(key).strip()
|
||||
value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value)
|
||||
return {item for item in re.split(r"[^A-Za-z0-9]+", value.lower()) if item}
|
||||
|
||||
|
||||
def is_sensitive_key(key: object) -> bool:
|
||||
tokens = sensitive_key_tokens(key)
|
||||
if not tokens:
|
||||
return False
|
||||
if tokens & {"authorization", "cookie", "credential", "credentials", "password", "passwd", "secret", "token"}:
|
||||
return True
|
||||
return bool({"api", "key"} <= tokens or {"access", "key"} <= tokens or {"private", "key"} <= tokens)
|
||||
|
||||
|
||||
def redact_secret_values(value: object, *, placeholder: str = "<redacted>") -> object:
|
||||
if isinstance(value, Mapping):
|
||||
result: dict[str, object] = {}
|
||||
for key, item in value.items():
|
||||
if is_sensitive_key(key):
|
||||
result[str(key)] = placeholder
|
||||
else:
|
||||
result[str(key)] = redact_secret_values(item, placeholder=placeholder)
|
||||
return result
|
||||
if isinstance(value, list):
|
||||
return [redact_secret_values(item, placeholder=placeholder) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def contains_plain_secret(value: object) -> bool:
|
||||
if not isinstance(value, Mapping):
|
||||
return False
|
||||
for key, item in value.items():
|
||||
normalized_key = str(key).strip().casefold().replace("-", "_")
|
||||
if normalized_key in {"credential_ref", "secret_ref", "secret_reference"}:
|
||||
continue
|
||||
if is_sensitive_key(key) and item not in (None, "", {"secret_ref": ""}):
|
||||
return True
|
||||
if isinstance(item, Mapping) and contains_plain_secret(item):
|
||||
return True
|
||||
return False
|
||||
@@ -43,9 +43,11 @@ def create_govoplan_app(
|
||||
|
||||
origins = [item.strip() for item in cors_origins if item.strip()]
|
||||
if origins:
|
||||
if "*" in origins:
|
||||
raise ValueError("CORS wildcard origin '*' cannot be used with credentialed requests; configure explicit origins.")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"] if "*" in origins else origins,
|
||||
allow_origins=origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
|
||||
@@ -18,6 +18,7 @@ class Settings(BaseSettings):
|
||||
tenant_db_url_template: str | None = Field(default=None, alias="TENANT_DB_URL_TEMPLATE")
|
||||
tenant_schema_template: str | None = Field(default=None, alias="TENANT_SCHEMA_TEMPLATE")
|
||||
enabled_modules: str = Field(default="tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops", alias="ENABLED_MODULES")
|
||||
migration_track: str = Field(default="release", alias="GOVOPLAN_MIGRATION_TRACK")
|
||||
redis_url: str = Field(default="redis://redis:6379/0", alias="REDIS_URL")
|
||||
celery_enabled: bool = Field(default=False, alias="CELERY_ENABLED")
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from govoplan_core.core.access import (
|
||||
AccessDecisionProvenance,
|
||||
AccessExplanationService,
|
||||
AccessGovernanceMaterializer,
|
||||
ResourceAccessExplanationProvider,
|
||||
AccessSemanticDirectory,
|
||||
AccessSubjectRef,
|
||||
AccountRef,
|
||||
@@ -75,8 +76,10 @@ from govoplan_core.core.campaigns import (
|
||||
CampaignPolicyContextProvider,
|
||||
CampaignRetentionProvider,
|
||||
)
|
||||
from govoplan_core.core.files import CAPABILITY_FILES_ACCESS, FileAccessProvider
|
||||
from govoplan_core.core.modules import ModuleContext, ModuleManifest
|
||||
from govoplan_core.core.organizations import CAPABILITY_ORGANIZATION_DIRECTORY, OrganizationDirectory as CoreOrganizationDirectory, OrganizationFunctionRef as CoreOrganizationFunctionRef
|
||||
from govoplan_core.core.idm import CAPABILITY_IDM_DIRECTORY, OrganizationFunctionAssignmentRef
|
||||
from govoplan_core.core.organizations import CAPABILITY_ORGANIZATION_DIRECTORY, OrganizationDirectory as CoreOrganizationDirectory, OrganizationFunctionRef as CoreOrganizationFunctionRef, OrganizationUnitRef as CoreOrganizationUnitRef
|
||||
from govoplan_core.core.registry import PlatformRegistry
|
||||
from govoplan_core.core.runtime import clear_runtime, configure_runtime
|
||||
from govoplan_core.db.base import Base
|
||||
@@ -85,6 +88,7 @@ from govoplan_core.server.config import GovoplanServerConfig
|
||||
from govoplan_core.tenancy.scope import create_scope_tables
|
||||
from govoplan_access.backend.db.models import (
|
||||
Account,
|
||||
ExternalFunctionRoleAssignment,
|
||||
Function,
|
||||
FunctionAssignment,
|
||||
FunctionRoleAssignment,
|
||||
@@ -237,6 +241,24 @@ class _FakeResourceAccessService:
|
||||
return AccessDecision(allowed=True)
|
||||
|
||||
|
||||
class _FakeResourceAccessExplanationProvider:
|
||||
def explain_resource_provenance(
|
||||
self,
|
||||
session: object,
|
||||
principal: PrincipalRef,
|
||||
*,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
action: str,
|
||||
):
|
||||
del session, principal, resource_type, action
|
||||
return (AccessDecisionProvenance(kind="resource", id=resource_id, label="Demo resource"),)
|
||||
|
||||
|
||||
class _FakeFileAccessProvider(_FakeResourceAccessExplanationProvider):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeAccessExplanationService:
|
||||
provenance = (
|
||||
AccessDecisionProvenance(kind="identity", id="identity-1", label="Demo Person"),
|
||||
@@ -479,6 +501,7 @@ class AccessContractTests(unittest.TestCase):
|
||||
self.assertEqual("access.administration", CAPABILITY_ACCESS_ADMINISTRATION)
|
||||
self.assertEqual("access.governanceMaterializer", CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER)
|
||||
self.assertEqual("campaigns.access", CAPABILITY_CAMPAIGNS_ACCESS)
|
||||
self.assertEqual("files.access", CAPABILITY_FILES_ACCESS)
|
||||
self.assertEqual("campaigns.deliveryTasks", CAPABILITY_CAMPAIGNS_DELIVERY_TASKS)
|
||||
self.assertEqual("campaigns.mailPolicyContext", CAPABILITY_CAMPAIGNS_MAIL_POLICY_CONTEXT)
|
||||
self.assertEqual("campaigns.policyContext", CAPABILITY_CAMPAIGNS_POLICY_CONTEXT)
|
||||
@@ -582,6 +605,8 @@ class AccessContractTests(unittest.TestCase):
|
||||
self.assertIsInstance(_FakeAccessSemanticDirectory(), AccessSemanticDirectory)
|
||||
self.assertIsInstance(_FakePermissionEvaluator(), PermissionEvaluator)
|
||||
self.assertIsInstance(_FakeResourceAccessService(), ResourceAccessService)
|
||||
self.assertIsInstance(_FakeResourceAccessExplanationProvider(), ResourceAccessExplanationProvider)
|
||||
self.assertIsInstance(_FakeFileAccessProvider(), FileAccessProvider)
|
||||
self.assertIsInstance(_FakeAccessExplanationService(), AccessExplanationService)
|
||||
self.assertIsInstance(_FakeTenantAccessProvisioner(), TenantAccessProvisioner)
|
||||
self.assertIsInstance(_FakeAccessAdministration(), AccessAdministration)
|
||||
@@ -1031,6 +1056,268 @@ class AccessContractTests(unittest.TestCase):
|
||||
clear_runtime()
|
||||
shutil.rmtree(root, ignore_errors=True)
|
||||
|
||||
def test_access_explanation_includes_idm_function_role_sources(self) -> None:
|
||||
root = Path(tempfile.mkdtemp(prefix="govoplan-access-explanation-"))
|
||||
try:
|
||||
tenant_id = "tenant-access-explanation"
|
||||
account_id = "account-access-explanation"
|
||||
user_id = "user-access-explanation"
|
||||
role_id = "role-access-explanation"
|
||||
function_id = "function-access-explanation"
|
||||
organization_unit_id = "unit-access-explanation"
|
||||
assignment_id = "assignment-access-explanation"
|
||||
identity_id = "identity-access-explanation"
|
||||
|
||||
assignment = OrganizationFunctionAssignmentRef(
|
||||
id=assignment_id,
|
||||
tenant_id=tenant_id,
|
||||
identity_id=identity_id,
|
||||
account_id=account_id,
|
||||
function_id=function_id,
|
||||
organization_unit_id=organization_unit_id,
|
||||
applies_to_subunits=True,
|
||||
source="direct",
|
||||
)
|
||||
|
||||
class FakeIdmDirectory:
|
||||
def get_organization_function_assignment(self, requested_assignment_id: str):
|
||||
return assignment if requested_assignment_id == assignment_id else None
|
||||
|
||||
def organization_function_assignments_for_identity(self, requested_identity_id: str, *, tenant_id: str | None = None):
|
||||
if requested_identity_id == identity_id and tenant_id in (None, assignment.tenant_id):
|
||||
return (assignment,)
|
||||
return ()
|
||||
|
||||
def organization_function_assignments_for_account(self, requested_account_id: str, *, tenant_id: str | None = None):
|
||||
if requested_account_id == account_id and tenant_id in (None, assignment.tenant_id):
|
||||
return (assignment,)
|
||||
return ()
|
||||
|
||||
class FakeOrganizationDirectory(CoreOrganizationDirectory):
|
||||
def get_organization_unit(self, requested_organization_unit_id: str):
|
||||
if requested_organization_unit_id != organization_unit_id:
|
||||
return None
|
||||
return CoreOrganizationUnitRef(
|
||||
id=organization_unit_id,
|
||||
tenant_id=tenant_id,
|
||||
slug="registry",
|
||||
name="Registry Office",
|
||||
)
|
||||
|
||||
def organization_units_for_tenant(self, requested_tenant_id: str):
|
||||
if requested_tenant_id != tenant_id:
|
||||
return ()
|
||||
unit = self.get_organization_unit(organization_unit_id)
|
||||
return (unit,) if unit is not None else ()
|
||||
|
||||
def get_function(self, requested_function_id: str):
|
||||
if requested_function_id != function_id:
|
||||
return None
|
||||
return CoreOrganizationFunctionRef(
|
||||
id=function_id,
|
||||
tenant_id=tenant_id,
|
||||
organization_unit_id=organization_unit_id,
|
||||
slug="registry-clerk",
|
||||
name="Registry Clerk",
|
||||
)
|
||||
|
||||
def functions_for_organization_unit(self, requested_organization_unit_id: str, *, include_subunits: bool = False):
|
||||
del include_subunits
|
||||
if requested_organization_unit_id != organization_unit_id:
|
||||
return ()
|
||||
function = self.get_function(function_id)
|
||||
return (function,) if function is not None else ()
|
||||
|
||||
idm_directory = FakeIdmDirectory()
|
||||
organization_directory = FakeOrganizationDirectory()
|
||||
|
||||
class FakeFileAccessProvider:
|
||||
def explain_resource_provenance(
|
||||
self,
|
||||
session: object,
|
||||
principal: PrincipalRef,
|
||||
*,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
action: str,
|
||||
):
|
||||
del session, principal, action
|
||||
if resource_type != "file":
|
||||
return ()
|
||||
return (
|
||||
AccessDecisionProvenance(
|
||||
kind="resource",
|
||||
id=resource_id,
|
||||
label="Registry.pdf",
|
||||
tenant_id=tenant_id,
|
||||
source="files.file",
|
||||
details={"resource_type": "file"},
|
||||
),
|
||||
)
|
||||
|
||||
resource_provider = FakeFileAccessProvider()
|
||||
|
||||
with temporary_database(f"sqlite:///{root / 'test.db'}") as database:
|
||||
create_scope_tables(database.engine)
|
||||
Base.metadata.create_all(bind=database.engine)
|
||||
with database.session() as session:
|
||||
tenant = Tenant(id=tenant_id, slug="access-explanation", name="Access Explanation", settings={})
|
||||
account = Account(
|
||||
id=account_id,
|
||||
email="explanation@example.test",
|
||||
normalized_email="explanation@example.test",
|
||||
display_name="Access Explanation User",
|
||||
)
|
||||
user = User(
|
||||
id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
email=account.email,
|
||||
display_name=account.display_name,
|
||||
settings={},
|
||||
mail_profile_policy={},
|
||||
)
|
||||
role = Role(
|
||||
id=role_id,
|
||||
tenant_id=tenant_id,
|
||||
slug="file-reader",
|
||||
name="File Reader",
|
||||
permissions=["files:file:read"],
|
||||
)
|
||||
mapping = ExternalFunctionRoleAssignment(
|
||||
tenant_id=tenant_id,
|
||||
source_module="organizations",
|
||||
function_id=function_id,
|
||||
role_id=role_id,
|
||||
)
|
||||
session.add_all([tenant, account, user, role, mapping])
|
||||
session.commit()
|
||||
|
||||
from govoplan_access.backend.explanation import SqlAccessExplanationService, build_user_access_explanation
|
||||
|
||||
explanation = build_user_access_explanation(
|
||||
session,
|
||||
user,
|
||||
idm_directory=idm_directory,
|
||||
organization_directory=organization_directory,
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(explanation.function_facts))
|
||||
self.assertEqual("Registry Clerk", explanation.function_facts[0].function_name)
|
||||
self.assertEqual((role_id,), explanation.function_facts[0].role_ids)
|
||||
source = next(item for item in explanation.role_sources if item.source_type == "idm_function_role")
|
||||
self.assertEqual("Registry Clerk", source.function_name)
|
||||
self.assertEqual("File Reader", source.role_name)
|
||||
scope = next(item for item in explanation.scopes if item.scope == "files:file:read")
|
||||
self.assertEqual(["idm_function_role"], [item.source_type for item in scope.sources])
|
||||
|
||||
service = SqlAccessExplanationService(
|
||||
idm_directory=idm_directory,
|
||||
organization_directory=organization_directory,
|
||||
resource_explanation_providers=(resource_provider,),
|
||||
)
|
||||
provenance = service.explain_scope_provenance(
|
||||
PrincipalRef(
|
||||
account_id=account_id,
|
||||
membership_id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
identity_id=identity_id,
|
||||
scopes=frozenset({"files:file:read"}),
|
||||
),
|
||||
"files:file:read",
|
||||
)
|
||||
self.assertTrue(any(item.kind == "function" and item.label == "Registry Clerk" for item in provenance))
|
||||
self.assertTrue(any(item.kind == "role" and item.label == "File Reader" for item in provenance))
|
||||
resource_provenance = service.explain_resource_provenance(
|
||||
PrincipalRef(
|
||||
account_id=account_id,
|
||||
membership_id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
identity_id=identity_id,
|
||||
scopes=frozenset({"files:file:read"}),
|
||||
),
|
||||
resource_type="file",
|
||||
resource_id="file-access-explanation",
|
||||
action="files:file:read",
|
||||
)
|
||||
self.assertTrue(any(item.kind == "resource" and item.id == "file-access-explanation" for item in resource_provenance))
|
||||
self.assertTrue(any(item.kind == "right" and item.id == "files:file:read" for item in resource_provenance))
|
||||
|
||||
registry = PlatformRegistry()
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
id="access",
|
||||
name="Access",
|
||||
version="test",
|
||||
capability_factories={CAPABILITY_ACCESS_EXPLANATION: lambda context: service},
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
id="idm",
|
||||
name="IDM",
|
||||
version="test",
|
||||
capability_factories={CAPABILITY_IDM_DIRECTORY: lambda context: idm_directory},
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
ModuleManifest(
|
||||
id="organizations",
|
||||
name="Organizations",
|
||||
version="test",
|
||||
capability_factories={CAPABILITY_ORGANIZATION_DIRECTORY: lambda context: organization_directory},
|
||||
)
|
||||
)
|
||||
context = ModuleContext(registry=registry, settings=object())
|
||||
registry.configure_capability_context(context)
|
||||
configure_runtime(context)
|
||||
|
||||
from govoplan_access.backend.api.v1.routes import router as admin_router
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(admin_router)
|
||||
|
||||
def principal_override() -> ApiPrincipal:
|
||||
with database.session() as session:
|
||||
account = session.get(Account, account_id)
|
||||
user = session.get(User, user_id)
|
||||
assert account is not None
|
||||
assert user is not None
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id=account_id,
|
||||
membership_id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
scopes=frozenset({"admin:users:read"}),
|
||||
),
|
||||
account=account,
|
||||
user=user,
|
||||
)
|
||||
|
||||
app.dependency_overrides[get_api_principal] = principal_override
|
||||
with TestClient(app) as client:
|
||||
response = client.get(f"/admin/users/{user_id}/access-explanation")
|
||||
self.assertEqual(200, response.status_code, response.text)
|
||||
payload = response.json()
|
||||
self.assertIn("files:file:read", payload["user"]["effective_scopes"])
|
||||
self.assertEqual("idm_function_role", payload["role_sources"][0]["source_type"])
|
||||
self.assertEqual("Registry Clerk", payload["function_facts"][0]["function_name"])
|
||||
resource_response = client.get(
|
||||
"/admin/access/resource-explanation",
|
||||
params={
|
||||
"user_id": user_id,
|
||||
"resource_type": "file",
|
||||
"resource_id": "file-access-explanation",
|
||||
"action": "files:file:read",
|
||||
},
|
||||
)
|
||||
self.assertEqual(200, resource_response.status_code, resource_response.text)
|
||||
resource_payload = resource_response.json()
|
||||
self.assertTrue(any(item["kind"] == "resource" and item["id"] == "file-access-explanation" for item in resource_payload["provenance"]))
|
||||
finally:
|
||||
clear_runtime()
|
||||
shutil.rmtree(root, ignore_errors=True)
|
||||
|
||||
def test_api_principal_dependency_uses_access_resolver_capability(self) -> None:
|
||||
root = Path(tempfile.mkdtemp(prefix="govoplan-access-contract-"))
|
||||
try:
|
||||
|
||||
@@ -33,13 +33,14 @@ from govoplan_core.db.bootstrap import bootstrap_dev_data
|
||||
from govoplan_core.db.session import configure_database, set_database
|
||||
from govoplan_core.core.change_sequence import decode_sequence_watermark, prune_sequence_entries
|
||||
from govoplan_core.core.pagination import encode_keyset_cursor, keyset_query_fingerprint
|
||||
from govoplan_core.tenancy.scope import create_scope_tables, scope_registry
|
||||
from govoplan_access.backend.permissions.catalog import permission_catalog as access_permission_catalog
|
||||
|
||||
_database = configure_database(os.environ["DATABASE_URL"])
|
||||
engine = _database.engine
|
||||
SessionLocal = _database.SessionLocal
|
||||
|
||||
from govoplan_core.server.app import app
|
||||
from govoplan_core.security.permissions import SYSTEM_SCOPES
|
||||
|
||||
|
||||
class ApiSmokeTests(unittest.TestCase):
|
||||
@@ -56,7 +57,9 @@ class ApiSmokeTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
set_database(_database)
|
||||
self.client.cookies.clear()
|
||||
scope_registry.metadata.drop_all(bind=engine)
|
||||
Base.metadata.drop_all(bind=engine)
|
||||
create_scope_tables(engine)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
shutil.rmtree(_TEST_ROOT / "files", ignore_errors=True)
|
||||
shutil.rmtree(_TEST_ROOT / "mock-mailbox", ignore_errors=True)
|
||||
@@ -5972,7 +5975,12 @@ class ApiSmokeTests(unittest.TestCase):
|
||||
auditor = next(item for item in roles.json()["roles"] if item["slug"] == "system_auditor")
|
||||
self.assertTrue(owner["is_builtin"])
|
||||
self.assertEqual(owner["user_assignments"], 1)
|
||||
self.assertEqual(owner["effective_permission_count"], len(SYSTEM_SCOPES))
|
||||
expected_system_permission_count = sum(
|
||||
1
|
||||
for permission in access_permission_catalog(include_legacy=False)
|
||||
if permission.level == "system"
|
||||
)
|
||||
self.assertEqual(owner["effective_permission_count"], expected_system_permission_count)
|
||||
self.assertEqual(owner["permissions"], ["system:*"])
|
||||
self.assertFalse(administrator["is_builtin"])
|
||||
self.assertFalse(auditor["is_builtin"])
|
||||
|
||||
@@ -133,6 +133,15 @@ class CoreEventTests(unittest.TestCase):
|
||||
self.assertEqual("request-123", response.json()["correlation_id"])
|
||||
self.assertEqual("request-123", response.headers["X-Correlation-ID"])
|
||||
|
||||
def test_app_rejects_wildcard_cors_with_credentials(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "wildcard origin"):
|
||||
create_govoplan_app(
|
||||
title="event test",
|
||||
version="test",
|
||||
registry=PlatformRegistry(),
|
||||
cors_origins=("*",),
|
||||
)
|
||||
|
||||
def test_audit_event_persists_trace_details_from_event_context(self) -> None:
|
||||
root = Path(tempfile.mkdtemp(prefix="govoplan-audit-trace-"))
|
||||
try:
|
||||
@@ -260,6 +269,8 @@ class CoreEventTests(unittest.TestCase):
|
||||
self.assertEqual("queued", payload["outcome"])
|
||||
self.assertEqual({"correlation_id": "corr-1", "causation_id": "cause-1"}, payload["_trace"])
|
||||
self.assertEqual("<redacted>", payload["options"]["api_key"])
|
||||
self.assertEqual("<redacted>", audit_operation_context(options={"bearer_token": "secret"})["options"]["bearer_token"])
|
||||
self.assertEqual("<redacted>", audit_operation_context(options={"clientSecret": "secret"})["options"]["clientSecret"])
|
||||
self.assertNotIn("token", payload["_trace"])
|
||||
self.assertNotIn("empty", payload)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user