Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e6f7c45f0a | |||
| 8aa1943581 | |||
| b9badc9153 | |||
| 9a0c467d55 | |||
| a00ef54821 | |||
| edb4687826 | |||
| fcfe0b69a3 | |||
| 715bdcbebe | |||
| 060f4da751 | |||
| c32951393a | |||
| 7b85d6deae | |||
| 2d2d9e7bc7 | |||
| dc1a250797 | |||
| 7b7cc8ada7 | |||
| 722c9e5d1c | |||
| 63e54a67be | |||
| 12b623bec9 | |||
| b788afcae1 | |||
| 2b0cdf13f3 |
28
.dockerignore
Normal file
28
.dockerignore
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
.git
|
||||||
|
.venv
|
||||||
|
node_modules
|
||||||
|
webui/node_modules
|
||||||
|
audit-reports
|
||||||
|
runtime
|
||||||
|
dist
|
||||||
|
build
|
||||||
|
coverage
|
||||||
|
htmlcov
|
||||||
|
__pycache__
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.log
|
||||||
|
.mypy_cache
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
.cache
|
||||||
|
.component-test-build
|
||||||
|
.module-test-build
|
||||||
|
.policy-test-build
|
||||||
|
.template-preview-test-build
|
||||||
|
.import-test-build
|
||||||
|
webui/.component-test-build
|
||||||
|
webui/.module-test-build
|
||||||
|
webui/.policy-test-build
|
||||||
|
webui/.template-preview-test-build
|
||||||
|
webui/.import-test-build
|
||||||
32
.env.example
32
.env.example
@@ -1,32 +0,0 @@
|
|||||||
# GovOPlaN self-hosted install configuration.
|
|
||||||
# Copy to a deployment-local .env or secret store. Do not commit populated secrets.
|
|
||||||
|
|
||||||
APP_ENV=production
|
|
||||||
GOVOPLAN_INSTALL_PROFILE=self-hosted
|
|
||||||
MASTER_KEY_B64=<generate-with-govoplan-config-env-template-generate-secrets>
|
|
||||||
|
|
||||||
DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan
|
|
||||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan
|
|
||||||
|
|
||||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,docs,ops
|
|
||||||
|
|
||||||
CELERY_ENABLED=true
|
|
||||||
REDIS_URL=redis://127.0.0.1:6379/0
|
|
||||||
CELERY_QUEUES=send_email,append_sent,default
|
|
||||||
|
|
||||||
CORS_ORIGINS=https://govoplan.example.org
|
|
||||||
AUTH_COOKIE_SECURE=true
|
|
||||||
AUTH_COOKIE_SAMESITE=lax
|
|
||||||
AUTH_COOKIE_DOMAIN=
|
|
||||||
|
|
||||||
FILE_STORAGE_BACKEND=local
|
|
||||||
FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files
|
|
||||||
FILE_STORAGE_LOCAL_FALLBACK_ROOTS=
|
|
||||||
|
|
||||||
DEV_AUTO_MIGRATE_ENABLED=false
|
|
||||||
DEV_BOOTSTRAP_ENABLED=false
|
|
||||||
DEV_MAILBOX_API_ENABLED=false
|
|
||||||
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/etc/govoplan/catalog-keyring.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
name: Dependency Audit
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
schedule:
|
|
||||||
- cron: "23 3 * * 1"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dependency-audit:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: "3.12"
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: "22"
|
|
||||||
- name: Configure SSH for release dependencies
|
|
||||||
env:
|
|
||||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
|
||||||
run: |
|
|
||||||
mkdir -p ~/.ssh
|
|
||||||
chmod 700 ~/.ssh
|
|
||||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
|
||||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
|
||||||
chmod 600 ~/.ssh/id_ed25519
|
|
||||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
|
||||||
chmod 600 ~/.ssh/known_hosts
|
|
||||||
- name: Install backend dev audit dependencies
|
|
||||||
run: |
|
|
||||||
python -m venv .venv
|
|
||||||
.venv/bin/python -m pip install --upgrade pip
|
|
||||||
.venv/bin/python -m pip install -r requirements-release.txt
|
|
||||||
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
|
|
||||||
- name: Install WebUI release dependencies
|
|
||||||
working-directory: webui
|
|
||||||
run: |
|
|
||||||
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); for (const key of ['dependencies','devDependencies','peerDependencies','optionalDependencies','overrides']) if (rel[key]) pkg[key]=rel[key]; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
|
||||||
rm -f package-lock.json
|
|
||||||
npm cache clean --force
|
|
||||||
npm install --prefer-online
|
|
||||||
- name: Run dependency audits
|
|
||||||
run: bash scripts/check-dependency-audits.sh
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
name: Module Matrix
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
module-matrix:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: "3.12"
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: "22"
|
|
||||||
- name: Configure SSH for release dependencies
|
|
||||||
env:
|
|
||||||
GOVOPLAN_RELEASE_SSH_KEY_B64: ${{ secrets.GOVOPLAN_RELEASE_SSH_KEY_B64 }}
|
|
||||||
run: |
|
|
||||||
mkdir -p ~/.ssh
|
|
||||||
chmod 700 ~/.ssh
|
|
||||||
if [ -z "${GOVOPLAN_RELEASE_SSH_KEY_B64:-}" ]; then
|
|
||||||
echo "GOVOPLAN_RELEASE_SSH_KEY_B64 secret is required for git+ssh release dependencies."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
printf '%s' "$GOVOPLAN_RELEASE_SSH_KEY_B64" | base64 -d > ~/.ssh/id_ed25519
|
|
||||||
chmod 600 ~/.ssh/id_ed25519
|
|
||||||
echo 'git.add-ideas.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDe48IOof2fJS1dTbJtLWQnWnr+JorZXKIFdOAM9ct8G' > ~/.ssh/known_hosts
|
|
||||||
chmod 600 ~/.ssh/known_hosts
|
|
||||||
- name: Install backend release dependencies
|
|
||||||
run: |
|
|
||||||
python -m venv .venv
|
|
||||||
.venv/bin/python -m pip install --upgrade pip
|
|
||||||
.venv/bin/python -m pip install -r requirements-release.txt
|
|
||||||
.venv/bin/python -m pip install '.[dev]'
|
|
||||||
- name: Install WebUI release dependencies with test scripts
|
|
||||||
working-directory: webui
|
|
||||||
run: |
|
|
||||||
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); for (const key of ['dependencies','devDependencies','peerDependencies','optionalDependencies','overrides']) if (rel[key]) pkg[key]=rel[key]; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
|
||||||
rm -f package-lock.json
|
|
||||||
npm cache clean --force
|
|
||||||
npm install --prefer-online
|
|
||||||
- name: Run module matrix and contract tests
|
|
||||||
run: bash scripts/check-module-matrix.sh
|
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -148,6 +148,9 @@ webui/.policy-test-build/
|
|||||||
webui/.template-preview-test-build/
|
webui/.template-preview-test-build/
|
||||||
webui/.import-test-build/
|
webui/.import-test-build/
|
||||||
|
|
||||||
|
# Security audit reports
|
||||||
|
audit-reports/
|
||||||
|
|
||||||
# ---> Python
|
# ---> Python
|
||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
10
AGENTS.md
10
AGENTS.md
@@ -19,9 +19,9 @@ Use targeted commands first:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
||||||
```
|
```
|
||||||
|
|
||||||
For WebUI checks:
|
For WebUI checks:
|
||||||
@@ -36,8 +36,8 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi
|
|||||||
Run the consolidated focused check when a change touches module discovery, optional integrations, shared mail components, or mailbox listing:
|
Run the consolidated focused check when a change touches module discovery, optional integrations, shared mail components, or mailbox listing:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./scripts/check-focused.sh
|
tools/checks/check-focused.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
## Working Rules
|
## Working Rules
|
||||||
|
|||||||
58
README.md
58
README.md
@@ -1,5 +1,13 @@
|
|||||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions/workflows/module-matrix.yml)
|
# govoplan-core
|
||||||
[](https://git.add-ideas.de/add-ideas/govoplan-core/actions/workflows/dependency-audit.yml)
|
|
||||||
|
<!-- govoplan-repository-type:start -->
|
||||||
|
**Repository type:** system (kernel).
|
||||||
|
<!-- govoplan-repository-type:end -->
|
||||||
|
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=module-matrix.yml&actor=0&status=0)
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=release-integration.yml&actor=0&status=0)
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=dependency-audit.yml&actor=0&status=0)
|
||||||
|
[](https://git.add-ideas.de/add-ideas/govoplan/actions?workflow=security-audit.yml&actor=0&status=0)
|
||||||
|
|
||||||
# govoplan-core
|
# govoplan-core
|
||||||
|
|
||||||
@@ -37,10 +45,12 @@ composition rules live in core only where they are stable kernel contracts.
|
|||||||
|
|
||||||
## Backend development
|
## Backend development
|
||||||
|
|
||||||
Create or activate the core virtual environment, then install core and sibling modules from this repository:
|
For whole-product development, create the virtualenv from the meta repository:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
|
python3 -m venv .venv
|
||||||
|
./.venv/bin/python -m pip install --upgrade pip
|
||||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -48,7 +58,7 @@ Run the platform server from core through the module-aware development runner. T
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m govoplan_core.devserver \
|
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||||
--host 127.0.0.1 \
|
--host 127.0.0.1 \
|
||||||
--port 8000
|
--port 8000
|
||||||
```
|
```
|
||||||
@@ -57,7 +67,7 @@ For example, to test campaign without files or mail:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \
|
ENABLED_MODULES=access,campaigns /mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver \
|
||||||
--host 127.0.0.1 \
|
--host 127.0.0.1 \
|
||||||
--port 8000
|
--port 8000
|
||||||
```
|
```
|
||||||
@@ -72,12 +82,29 @@ To run the production-like local profile with PostgreSQL, Redis, a Celery
|
|||||||
worker, explicit module configuration, and persistent local file storage:
|
worker, explicit module configuration, and persistent local file storage:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
scripts/launch-production-like-dev.sh
|
tools/launch/launch-production-like-dev.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
See [dev/production-like/README.md](dev/production-like/README.md) for ports,
|
See `/mnt/DATA/git/govoplan/dev/production-like/README.md` for ports,
|
||||||
environment overrides, and cleanup commands.
|
environment overrides, and cleanup commands. Core keeps wrapper commands during
|
||||||
|
the migration, but whole-product profiles are owned by the meta repository.
|
||||||
|
|
||||||
|
## Security audit
|
||||||
|
|
||||||
|
The repository includes a containerized audit toolbox for SAST, secret scanning,
|
||||||
|
dependency checks, filesystem misconfiguration scans, duplication, and complexity
|
||||||
|
reports. See [SECURITY_AUDIT.md](docs/SECURITY_AUDIT.md) for the operating model.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan
|
||||||
|
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||||
|
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||||
|
```
|
||||||
|
|
||||||
|
CI runs the `ci` profile in report-only mode and uploads `audit-reports/` as an
|
||||||
|
artifact. Once the baseline is clean, set `SECURITY_AUDIT_FAIL_ON_FINDINGS=1`
|
||||||
|
or pass `--strict` locally to turn findings into a failing gate.
|
||||||
|
|
||||||
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
||||||
|
|
||||||
@@ -85,20 +112,23 @@ To verify the effective runtime paths and bootstrap behavior without starting uv
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
```
|
```
|
||||||
|
|
||||||
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once.
|
||||||
|
|
||||||
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
The meta repository owns whole-product `requirements-dev.txt`,
|
||||||
|
`requirements-release.txt`, and the root `.env.example` operator template. Core
|
||||||
|
keeps package metadata and runtime commands. See
|
||||||
|
[RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||||
|
|
||||||
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
||||||
For self-hosted config bootstrap and validation:
|
For self-hosted config bootstrap and validation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config env-template --profile self-hosted --generate-secrets
|
||||||
./.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
/mnt/DATA/git/govoplan/.venv/bin/python -m govoplan_core.commands.config validate --profile self-hosted
|
||||||
```
|
```
|
||||||
|
|
||||||
## WebUI development
|
## WebUI development
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
[alembic]
|
[alembic]
|
||||||
script_location = alembic
|
script_location = alembic
|
||||||
|
path_separator = os
|
||||||
prepend_sys_path = .
|
prepend_sys_path = .
|
||||||
sqlalchemy.url = sqlite:///./runtime/multimailer-dev.db
|
sqlalchemy.url = sqlite:///./runtime/govoplan-dev.db
|
||||||
|
|
||||||
[loggers]
|
[loggers]
|
||||||
keys = root,sqlalchemy,alembic
|
keys = root,sqlalchemy,alembic
|
||||||
|
|||||||
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
52
alembic/dev_versions/0f1e2d3c4b5a_audit_outbox_events.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
"""add audit outbox events to detailed dev track
|
||||||
|
|
||||||
|
Revision ID: 0f1e2d3c4b5a
|
||||||
|
Revises: 4f2a9c8e7b6d
|
||||||
|
Create Date: 2026-07-11 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "0f1e2d3c4b5a"
|
||||||
|
down_revision = "4f2a9c8e7b6d"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"audit_outbox_events",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("event_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("event_type", sa.String(length=200), nullable=False),
|
||||||
|
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("correlation_id", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("causation_id", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("classification", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("payload", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=20), nullable=False),
|
||||||
|
sa.Column("attempts", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("dispatched_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_outbox_events")),
|
||||||
|
sa.UniqueConstraint("event_id", name="uq_audit_outbox_events_event_id"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_audit_outbox_events_correlation_id", "audit_outbox_events", ["correlation_id"], unique=False)
|
||||||
|
op.create_index("ix_audit_outbox_events_event_type", "audit_outbox_events", ["event_type"], unique=False)
|
||||||
|
op.create_index(op.f("ix_audit_outbox_events_status"), "audit_outbox_events", ["status"], unique=False)
|
||||||
|
op.create_index(
|
||||||
|
"ix_audit_outbox_events_status_next_attempt_at",
|
||||||
|
"audit_outbox_events",
|
||||||
|
["status", "next_attempt_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("audit_outbox_events")
|
||||||
@@ -30,10 +30,14 @@ _TABLE_RENAMES = (
|
|||||||
("governance_templates", "admin_governance_templates"),
|
("governance_templates", "admin_governance_templates"),
|
||||||
("governance_template_assignments", "admin_governance_template_assignments"),
|
("governance_template_assignments", "admin_governance_template_assignments"),
|
||||||
)
|
)
|
||||||
|
_KNOWN_TABLE_NAMES = {name for pair in _TABLE_RENAMES for name in pair}
|
||||||
|
|
||||||
|
|
||||||
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one())
|
if table_name not in _KNOWN_TABLE_NAMES:
|
||||||
|
raise RuntimeError(f"Unexpected table name: {table_name}")
|
||||||
|
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||||
|
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||||
|
|
||||||
|
|
||||||
def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None:
|
def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None:
|
||||||
@@ -19,11 +19,14 @@ LEGACY_SCOPE_TABLE = "tenancy_tenants"
|
|||||||
CORE_SCOPE_TABLE = "core_scopes"
|
CORE_SCOPE_TABLE = "core_scopes"
|
||||||
LEGACY_SLUG_INDEX = "ix_tenancy_tenants_slug"
|
LEGACY_SLUG_INDEX = "ix_tenancy_tenants_slug"
|
||||||
CORE_SLUG_INDEX = "ix_core_scopes_slug"
|
CORE_SLUG_INDEX = "ix_core_scopes_slug"
|
||||||
|
_KNOWN_SCOPE_TABLES = {LEGACY_SCOPE_TABLE, CORE_SCOPE_TABLE}
|
||||||
|
|
||||||
|
|
||||||
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||||
|
if table_name not in _KNOWN_SCOPE_TABLES:
|
||||||
|
raise RuntimeError(f"Unexpected scope table name: {table_name}")
|
||||||
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||||
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one())
|
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||||
|
|
||||||
|
|
||||||
def _scope_tables(bind: sa.Connection) -> set[str]:
|
def _scope_tables(bind: sa.Connection) -> set[str]:
|
||||||
@@ -16,6 +16,7 @@ revision = "9d0e1f2a3b4c"
|
|||||||
down_revision = "8c9d0e1f2a3b"
|
down_revision = "8c9d0e1f2a3b"
|
||||||
branch_labels = None
|
branch_labels = None
|
||||||
depends_on = None
|
depends_on = None
|
||||||
|
_RECONCILE_CREATE_ALL_TABLES = ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings")
|
||||||
|
|
||||||
|
|
||||||
def _now() -> datetime:
|
def _now() -> datetime:
|
||||||
@@ -28,9 +29,10 @@ def upgrade() -> None:
|
|||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
# Reconcile only the empty create_all shape for the newly introduced tables.
|
# Reconcile only the empty create_all shape for the newly introduced tables.
|
||||||
for table_name in ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings"):
|
for table_name in _RECONCILE_CREATE_ALL_TABLES:
|
||||||
if table_name in tables:
|
if table_name in tables:
|
||||||
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
quoted = bind.dialect.identifier_preparer.quote(table_name)
|
||||||
|
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one() # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||||
if count:
|
if count:
|
||||||
raise RuntimeError(f"Cannot reconcile non-empty create_all table {table_name}")
|
raise RuntimeError(f"Cannot reconcile non-empty create_all table {table_name}")
|
||||||
op.drop_table(table_name)
|
op.drop_table(table_name)
|
||||||
@@ -49,7 +49,7 @@ def upgrade() -> None:
|
|||||||
placeholders = ", ".join(f":action_{index}" for index, _ in enumerate(SYSTEM_ACTIONS))
|
placeholders = ", ".join(f":action_{index}" for index, _ in enumerate(SYSTEM_ACTIONS))
|
||||||
params = {f"action_{index}": action for index, action in enumerate(SYSTEM_ACTIONS)}
|
params = {f"action_{index}": action for index, action in enumerate(SYSTEM_ACTIONS)}
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text(f"UPDATE audit_log SET scope = 'system' WHERE action IN ({placeholders})"),
|
sa.text(f"UPDATE audit_log SET scope = 'system' WHERE action IN ({placeholders})"), # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
bind.execute(sa.text("UPDATE audit_log SET scope = 'tenant' WHERE scope IS NULL OR scope NOT IN ('tenant', 'system')"))
|
bind.execute(sa.text("UPDATE audit_log SET scope = 'tenant' WHERE scope IS NULL OR scope NOT IN ('tenant', 'system')"))
|
||||||
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
892
alembic/versions/4f2a9c8e7b6d_v017_core_baseline.py
Normal file
@@ -0,0 +1,892 @@
|
|||||||
|
"""v0.1.7 core baseline
|
||||||
|
|
||||||
|
Revision ID: 4f2a9c8e7b6d
|
||||||
|
Revises: None
|
||||||
|
Create Date: 2026-07-11 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from uuid import uuid4
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '4f2a9c8e7b6d'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_core_defaults() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
now = _now()
|
||||||
|
if not bind.execute(sa.text("SELECT 1 FROM core_system_settings WHERE id = 'global'")).first():
|
||||||
|
settings_table = sa.table(
|
||||||
|
"core_system_settings",
|
||||||
|
sa.column("id", sa.String),
|
||||||
|
sa.column("default_locale", sa.String),
|
||||||
|
sa.column("allow_tenant_custom_groups", sa.Boolean),
|
||||||
|
sa.column("allow_tenant_custom_roles", sa.Boolean),
|
||||||
|
sa.column("allow_tenant_api_keys", sa.Boolean),
|
||||||
|
sa.column("settings", sa.JSON),
|
||||||
|
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
|
)
|
||||||
|
bind.execute(
|
||||||
|
settings_table.insert().values({
|
||||||
|
"id": "global",
|
||||||
|
"default_locale": "en",
|
||||||
|
"allow_tenant_custom_groups": True,
|
||||||
|
"allow_tenant_custom_roles": True,
|
||||||
|
"allow_tenant_api_keys": True,
|
||||||
|
"settings": {},
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
system_roles = {
|
||||||
|
"system_owner": {
|
||||||
|
"name": "System owner",
|
||||||
|
"description": "Protected full instance-wide administration.",
|
||||||
|
"permissions": ["system:*"],
|
||||||
|
"is_builtin": True,
|
||||||
|
},
|
||||||
|
"system_admin": {
|
||||||
|
"name": "System administrator",
|
||||||
|
"description": "Manage the instance without granting the protected System owner role.",
|
||||||
|
"permissions": [
|
||||||
|
"system:tenants:read", "system:tenants:create", "system:tenants:update", "system:tenants:suspend",
|
||||||
|
"system:accounts:read", "system:accounts:create", "system:accounts:update", "system:accounts:suspend",
|
||||||
|
"system:roles:read", "system:roles:write", "system:roles:assign", "system:access:read", "system:access:assign",
|
||||||
|
"system:audit:read", "system:settings:read", "system:settings:write", "system:governance:read", "system:governance:write",
|
||||||
|
],
|
||||||
|
"is_builtin": False,
|
||||||
|
},
|
||||||
|
"system_auditor": {
|
||||||
|
"name": "System auditor",
|
||||||
|
"description": "Read-only access to system administration and audit.",
|
||||||
|
"permissions": [
|
||||||
|
"system:tenants:read", "system:accounts:read", "system:roles:read", "system:access:read",
|
||||||
|
"system:audit:read", "system:settings:read", "system:governance:read",
|
||||||
|
],
|
||||||
|
"is_builtin": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
roles_table = sa.table(
|
||||||
|
"access_roles",
|
||||||
|
sa.column("id", sa.String),
|
||||||
|
sa.column("tenant_id", sa.String),
|
||||||
|
sa.column("slug", sa.String),
|
||||||
|
sa.column("name", sa.String),
|
||||||
|
sa.column("description", sa.Text),
|
||||||
|
sa.column("permissions", sa.JSON),
|
||||||
|
sa.column("is_builtin", sa.Boolean),
|
||||||
|
sa.column("is_assignable", sa.Boolean),
|
||||||
|
sa.column("system_template_id", sa.String),
|
||||||
|
sa.column("system_required", sa.Boolean),
|
||||||
|
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
|
)
|
||||||
|
for slug, role in system_roles.items():
|
||||||
|
if bind.execute(
|
||||||
|
sa.text("SELECT 1 FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"),
|
||||||
|
{"slug": slug},
|
||||||
|
).first():
|
||||||
|
continue
|
||||||
|
bind.execute(
|
||||||
|
roles_table.insert().values(
|
||||||
|
id=str(uuid4()),
|
||||||
|
tenant_id=None,
|
||||||
|
slug=slug,
|
||||||
|
name=role["name"],
|
||||||
|
description=role["description"],
|
||||||
|
permissions=role["permissions"],
|
||||||
|
is_builtin=bool(role["is_builtin"]),
|
||||||
|
is_assignable=True,
|
||||||
|
system_template_id=None,
|
||||||
|
system_required=False,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table('core_scopes',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('settings', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('allow_custom_groups', sa.Boolean(), nullable=True),
|
||||||
|
sa.Column('allow_custom_roles', sa.Boolean(), nullable=True),
|
||||||
|
sa.Column('allow_api_keys', sa.Boolean(), nullable=True),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_scopes'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_core_scopes_slug'), 'core_scopes', ['slug'], unique=True)
|
||||||
|
op.create_table('access_accounts',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('email', sa.String(length=320), nullable=False),
|
||||||
|
sa.Column('normalized_email', sa.String(length=320), nullable=False),
|
||||||
|
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('password_reset_required', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_accounts'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_accounts_normalized_email'), 'access_accounts', ['normalized_email'], unique=True)
|
||||||
|
op.create_table('access_groups',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('settings', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_groups')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_groups_system_template_id'), 'access_groups', ['system_template_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('access_roles',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('is_builtin', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('is_assignable', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('system_template_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('system_required', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_roles')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_roles_system_template_id'), 'access_roles', ['system_template_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False)
|
||||||
|
op.create_index('uq_roles_system_slug', 'access_roles', ['slug'], unique=True, sqlite_where=sa.text('tenant_id IS NULL'), postgresql_where=sa.text('tenant_id IS NULL'))
|
||||||
|
op.create_table('admin_governance_templates',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('kind', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_templates')),
|
||||||
|
sa.UniqueConstraint('kind', 'slug', name='uq_governance_templates_kind_slug')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_admin_governance_templates_kind'), 'admin_governance_templates', ['kind'], unique=False)
|
||||||
|
op.create_table('attachment_blobs',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('mime_type', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('storage_bucket', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
||||||
|
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('audit_outbox_events',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('event_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('event_type', sa.String(length=200), nullable=False),
|
||||||
|
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('correlation_id', sa.String(length=128), nullable=True),
|
||||||
|
sa.Column('causation_id', sa.String(length=128), nullable=True),
|
||||||
|
sa.Column('classification', sa.String(length=40), nullable=False),
|
||||||
|
sa.Column('payload', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('status', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('attempts', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('dispatched_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_outbox_events')),
|
||||||
|
sa.UniqueConstraint('event_id', name='uq_audit_outbox_events_event_id')
|
||||||
|
)
|
||||||
|
op.create_index('ix_audit_outbox_events_correlation_id', 'audit_outbox_events', ['correlation_id'], unique=False)
|
||||||
|
op.create_index('ix_audit_outbox_events_event_type', 'audit_outbox_events', ['event_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_outbox_events_status'), 'audit_outbox_events', ['status'], unique=False)
|
||||||
|
op.create_index('ix_audit_outbox_events_status_next_attempt_at', 'audit_outbox_events', ['status', 'next_attempt_at'], unique=False)
|
||||||
|
op.create_table('core_change_sequence',
|
||||||
|
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||||
|
sa.Column('resource_type', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('resource_id', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('operation', sa.String(length=30), nullable=False),
|
||||||
|
sa.Column('actor_type', sa.String(length=30), nullable=True),
|
||||||
|
sa.Column('actor_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('payload', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_actor_id'), 'core_change_sequence', ['actor_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_actor_type'), 'core_change_sequence', ['actor_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_collection'), 'core_change_sequence', ['collection'], unique=False)
|
||||||
|
op.create_index('ix_core_change_sequence_collection_id', 'core_change_sequence', ['collection', 'id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_created_at'), 'core_change_sequence', ['created_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_module_id'), 'core_change_sequence', ['module_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_operation'), 'core_change_sequence', ['operation'], unique=False)
|
||||||
|
op.create_index('ix_core_change_sequence_resource', 'core_change_sequence', ['module_id', 'resource_type', 'resource_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_resource_id'), 'core_change_sequence', ['resource_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_resource_type'), 'core_change_sequence', ['resource_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_core_change_sequence_tenant_id'), 'core_change_sequence', ['tenant_id'], unique=False)
|
||||||
|
op.create_index('ix_core_change_sequence_tenant_module_id', 'core_change_sequence', ['tenant_id', 'module_id', 'id'], unique=False)
|
||||||
|
op.create_table('core_change_sequence_retention_floor',
|
||||||
|
sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), autoincrement=True, nullable=False),
|
||||||
|
sa.Column('tenant_key', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('module_id', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('collection', sa.String(length=150), nullable=False),
|
||||||
|
sa.Column('min_valid_sequence', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_change_sequence_retention_floor')),
|
||||||
|
sa.UniqueConstraint('tenant_key', 'module_id', 'collection', name='uq_core_change_sequence_retention_scope')
|
||||||
|
)
|
||||||
|
op.create_index('ix_core_change_sequence_retention_scope', 'core_change_sequence_retention_floor', ['tenant_key', 'module_id', 'collection'], unique=False)
|
||||||
|
op.create_table('core_system_settings',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('default_locale', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('allow_tenant_custom_groups', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('allow_tenant_custom_roles', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('allow_tenant_api_keys', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('settings', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_core_system_settings'))
|
||||||
|
)
|
||||||
|
op.create_table('file_blobs',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('storage_backend', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('storage_bucket', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('ref_count', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('retained_until', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_blobs')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'checksum_sha256', 'size_bytes', name='uq_file_blobs_tenant_checksum_size')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_file_blobs_checksum_sha256'), 'file_blobs', ['checksum_sha256'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_blobs_tenant_id'), 'file_blobs', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('access_group_role_assignments',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_group_role_assignments_group_id_access_groups'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_group_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_group_role_assignments')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'group_id', 'role_id', name='uq_group_role_assignments')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_group_role_assignments_group_id'), 'access_group_role_assignments', ['group_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_group_role_assignments_role_id'), 'access_group_role_assignments', ['role_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_group_role_assignments_tenant_id'), 'access_group_role_assignments', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('access_system_role_assignments',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_system_role_assignments_account_id_access_accounts'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_system_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_system_role_assignments')),
|
||||||
|
sa.UniqueConstraint('account_id', 'role_id', name='uq_system_role_assignments')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_system_role_assignments_account_id'), 'access_system_role_assignments', ['account_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_system_role_assignments_role_id'), 'access_system_role_assignments', ['role_id'], unique=False)
|
||||||
|
op.create_table('access_users',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('email', sa.String(length=320), nullable=False),
|
||||||
|
sa.Column('display_name', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('auth_provider', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('password_hash', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('settings', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_users_account_id_access_accounts'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_users')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'account_id', name='uq_users_tenant_account'),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_users_account_id'), 'access_users', ['account_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('admin_governance_template_assignments',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('template_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('mode', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['template_id'], ['admin_governance_templates.id'], name=op.f('fk_admin_governance_template_assignments_template_id_admin_governance_templates'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_admin_governance_template_assignments')),
|
||||||
|
sa.UniqueConstraint('template_id', 'tenant_id', name='uq_governance_template_tenant')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_admin_governance_template_assignments_template_id'), 'admin_governance_template_assignments', ['template_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_admin_governance_template_assignments_tenant_id'), 'admin_governance_template_assignments', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('access_api_keys',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('prefix', sa.String(length=16), nullable=False),
|
||||||
|
sa.Column('key_hash', sa.String(length=128), nullable=False),
|
||||||
|
sa.Column('scopes', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_api_keys_user_id_access_users'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_api_keys'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False)
|
||||||
|
op.create_table('access_auth_sessions',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('account_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('token_hash', sa.String(length=128), nullable=False),
|
||||||
|
sa.Column('csrf_token_hash', sa.String(length=128), nullable=True),
|
||||||
|
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('last_seen_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('user_agent', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('ip_address', sa.String(length=100), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['account_id'], ['access_accounts.id'], name=op.f('fk_access_auth_sessions_account_id_access_accounts'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_auth_sessions_user_id_access_users'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_auth_sessions'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_account_id'), 'access_auth_sessions', ['account_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_expires_at'), 'access_auth_sessions', ['expires_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_revoked_at'), 'access_auth_sessions', ['revoked_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_tenant_id'), 'access_auth_sessions', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_token_hash'), 'access_auth_sessions', ['token_hash'], unique=True)
|
||||||
|
op.create_index(op.f('ix_access_auth_sessions_user_id'), 'access_auth_sessions', ['user_id'], unique=False)
|
||||||
|
op.create_table('access_user_group_memberships',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('group_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['group_id'], ['access_groups.id'], name=op.f('fk_access_user_group_memberships_group_id_access_groups'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_group_memberships_user_id_access_users'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_group_memberships')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'user_id', 'group_id', name='uq_user_group_memberships')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_user_group_memberships_group_id'), 'access_user_group_memberships', ['group_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_user_group_memberships_tenant_id'), 'access_user_group_memberships', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_user_group_memberships_user_id'), 'access_user_group_memberships', ['user_id'], unique=False)
|
||||||
|
op.create_table('access_user_role_assignments',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('role_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['role_id'], ['access_roles.id'], name=op.f('fk_access_user_role_assignments_role_id_access_roles'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_access_user_role_assignments_user_id_access_users'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_access_user_role_assignments')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'user_id', 'role_id', name='uq_user_role_assignments')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_access_user_role_assignments_role_id'), 'access_user_role_assignments', ['role_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_user_role_assignments_tenant_id'), 'access_user_role_assignments', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_access_user_role_assignments_user_id'), 'access_user_role_assignments', ['user_id'], unique=False)
|
||||||
|
op.create_table('campaigns',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('external_id', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('settings', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('mail_profile_policy', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_campaigns_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_campaigns_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaigns_created_by_user_id'), 'campaigns', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaigns_external_id'), 'campaigns', ['external_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaigns_owner_group_id'), 'campaigns', ['owner_group_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaigns_owner_user_id'), 'campaigns', ['owner_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaigns_status'), 'campaigns', ['status'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaigns_tenant_id'), 'campaigns', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('file_assets',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('display_path', sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_assets_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_assets_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_assets_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_assets'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_file_assets_created_by_user_id'), 'file_assets', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_current_version_id'), 'file_assets', ['current_version_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_deleted_at'), 'file_assets', ['deleted_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_display_path'), 'file_assets', ['display_path'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_filename'), 'file_assets', ['filename'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_owner_group_id'), 'file_assets', ['owner_group_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_owner_type'), 'file_assets', ['owner_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_owner_user_id'), 'file_assets', ['owner_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_assets_tenant_id'), 'file_assets', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('file_folders',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('owner_type', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('owner_group_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('path', sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_folders_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_group_id'], ['access_groups.id'], name=op.f('fk_file_folders_owner_group_id_access_groups'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_file_folders_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_folders'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_file_folders_created_by_user_id'), 'file_folders', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_deleted_at'), 'file_folders', ['deleted_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_owner_group_id'), 'file_folders', ['owner_group_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_owner_type'), 'file_folders', ['owner_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_owner_user_id'), 'file_folders', ['owner_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_path'), 'file_folders', ['path'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_folders_tenant_id'), 'file_folders', ['tenant_id'], unique=False)
|
||||||
|
op.create_index('uq_file_folders_active_group_path', 'file_folders', ['tenant_id', 'owner_group_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'group' AND deleted_at IS NULL"))
|
||||||
|
op.create_index('uq_file_folders_active_user_path', 'file_folders', ['tenant_id', 'owner_user_id', 'path'], unique=True, sqlite_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"), postgresql_where=sa.text("owner_type = 'user' AND deleted_at IS NULL"))
|
||||||
|
op.create_table('mail_server_profiles',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('scope_type', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('scope_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('description', sa.Text(), nullable=True),
|
||||||
|
sa.Column('is_active', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('smtp_config', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('smtp_username', sa.String(length=320), nullable=True),
|
||||||
|
sa.Column('smtp_password_encrypted', sa.Text(), nullable=True),
|
||||||
|
sa.Column('imap_config', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('imap_username', sa.String(length=320), nullable=True),
|
||||||
|
sa.Column('imap_password_encrypted', sa.Text(), nullable=True),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('updated_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['updated_by_user_id'], ['access_users.id'], name=op.f('fk_mail_server_profiles_updated_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_mail_server_profiles')),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_mail_server_profiles_tenant_slug')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_created_by_user_id'), 'mail_server_profiles', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_is_active'), 'mail_server_profiles', ['is_active'], unique=False)
|
||||||
|
op.create_index('ix_mail_server_profiles_scope', 'mail_server_profiles', ['scope_type', 'scope_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_scope_id'), 'mail_server_profiles', ['scope_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_scope_type'), 'mail_server_profiles', ['scope_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_tenant_id'), 'mail_server_profiles', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_mail_server_profiles_updated_by_user_id'), 'mail_server_profiles', ['updated_by_user_id'], unique=False)
|
||||||
|
op.create_table('attachment_instances',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('owner_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('logical_name', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('filename', sa.String(length=500), nullable=False),
|
||||||
|
sa.Column('tags', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('metadata', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_attachment_instances_campaign_id'), 'attachment_instances', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_attachment_instances_owner_user_id'), 'attachment_instances', ['owner_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_attachment_instances_tenant_id'), 'attachment_instances', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('audit_log',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('scope', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('api_key_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('action', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('object_type', sa.String(length=100), nullable=True),
|
||||||
|
sa.Column('object_id', sa.String(length=100), nullable=True),
|
||||||
|
sa.Column('details', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_access_api_keys'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_api_key_id'), 'audit_log', ['api_key_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_object_id'), 'audit_log', ['object_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_object_type'), 'audit_log', ['object_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_scope'), 'audit_log', ['scope'], unique=False)
|
||||||
|
op.create_index('ix_audit_log_scope_created_at', 'audit_log', ['scope', 'created_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_tenant_id'), 'audit_log', ['tenant_id'], unique=False)
|
||||||
|
op.create_index('ix_audit_log_tenant_scope_created_at', 'audit_log', ['tenant_id', 'scope', 'created_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_log_user_id'), 'audit_log', ['user_id'], unique=False)
|
||||||
|
op.create_table('campaign_shares',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_shares_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_shares')),
|
||||||
|
sa.UniqueConstraint('campaign_id', 'target_type', 'target_id', name='uq_campaign_share_target')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_campaign_id'), 'campaign_shares', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_created_by_user_id'), 'campaign_shares', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_revoked_at'), 'campaign_shares', ['revoked_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_target_id'), 'campaign_shares', ['target_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_target_type'), 'campaign_shares', ['target_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_shares_tenant_id'), 'campaign_shares', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('campaign_versions',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('raw_json', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('schema_version', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('source_filename', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('source_base_path', sa.String(length=1000), nullable=True),
|
||||||
|
sa.Column('workflow_state', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('current_flow', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('current_step', sa.String(length=100), nullable=True),
|
||||||
|
sa.Column('is_complete', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('editor_state', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('autosaved_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('locked_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('user_lock_state', sa.String(length=20), nullable=True),
|
||||||
|
sa.Column('user_locked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('user_locked_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('validation_summary', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('build_summary', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('execution_snapshot', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('execution_snapshot_hash', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('execution_snapshot_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_versions_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['user_locked_by_user_id'], ['access_users.id'], name=op.f('fk_campaign_versions_user_locked_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_versions')),
|
||||||
|
sa.UniqueConstraint('campaign_id', 'version_number', name='uq_campaign_versions_campaign_number')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_campaign_id'), 'campaign_versions', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_current_flow'), 'campaign_versions', ['current_flow'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_execution_snapshot_hash'), 'campaign_versions', ['execution_snapshot_hash'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_locked_by_user_id'), 'campaign_versions', ['locked_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_user_lock_state'), 'campaign_versions', ['user_lock_state'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_user_locked_by_user_id'), 'campaign_versions', ['user_locked_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_versions_workflow_state'), 'campaign_versions', ['workflow_state'], unique=False)
|
||||||
|
op.create_table('file_shares',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('target_type', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('target_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('permission', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_shares_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_shares_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_shares')),
|
||||||
|
sa.UniqueConstraint('file_asset_id', 'target_type', 'target_id', 'revoked_at', name='uq_file_shares_active_target')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_file_shares_created_by_user_id'), 'file_shares', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_shares_file_asset_id'), 'file_shares', ['file_asset_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_shares_revoked_at'), 'file_shares', ['revoked_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_shares_target_id'), 'file_shares', ['target_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_shares_target_type'), 'file_shares', ['target_type'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_shares_tenant_id'), 'file_shares', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('file_versions',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('blob_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('version_number', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('filename_at_upload', sa.String(length=500), nullable=False),
|
||||||
|
sa.Column('display_path_at_upload', sa.String(length=1000), nullable=False),
|
||||||
|
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('created_by_user_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['blob_id'], ['file_blobs.id'], name=op.f('fk_file_versions_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||||
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_file_versions_created_by_user_id_access_users'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_file_versions_file_asset_id_file_assets'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_file_versions')),
|
||||||
|
sa.UniqueConstraint('file_asset_id', 'version_number', name='uq_file_versions_asset_number')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_file_versions_blob_id'), 'file_versions', ['blob_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_versions_checksum_sha256'), 'file_versions', ['checksum_sha256'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_versions_created_by_user_id'), 'file_versions', ['created_by_user_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_versions_file_asset_id'), 'file_versions', ['file_asset_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_file_versions_tenant_id'), 'file_versions', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('campaign_jobs',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('entry_index', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('recipient_email', sa.String(length=320), nullable=True),
|
||||||
|
sa.Column('subject', sa.String(length=998), nullable=True),
|
||||||
|
sa.Column('message_id_header', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('eml_storage_key', sa.String(length=1000), nullable=True),
|
||||||
|
sa.Column('eml_local_path', sa.String(length=1000), nullable=True),
|
||||||
|
sa.Column('eml_size_bytes', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('eml_sha256', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('build_status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('validation_status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('queue_status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('send_status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('imap_status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('attempt_count', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
|
sa.Column('queued_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('claimed_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('smtp_started_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('outcome_unknown_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('sent_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('resolved_recipients', sa.JSON(), nullable=True),
|
||||||
|
sa.Column('resolved_attachments', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('issues_snapshot', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
||||||
|
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_build_status'), 'campaign_jobs', ['build_status'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_campaign_id'), 'campaign_jobs', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_campaign_version_id'), 'campaign_jobs', ['campaign_version_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_claim_token'), 'campaign_jobs', ['claim_token'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_eml_sha256'), 'campaign_jobs', ['eml_sha256'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_entry_id'), 'campaign_jobs', ['entry_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_imap_status'), 'campaign_jobs', ['imap_status'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_queue_status'), 'campaign_jobs', ['queue_status'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_recipient_email'), 'campaign_jobs', ['recipient_email'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_send_status'), 'campaign_jobs', ['send_status'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_tenant_id'), 'campaign_jobs', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_jobs_validation_status'), 'campaign_jobs', ['validation_status'], unique=False)
|
||||||
|
op.create_table('campaign_attachment_uses',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_version_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_job_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('entry_index', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('entry_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('file_asset_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('file_version_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('file_blob_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('filename_used', sa.String(length=500), nullable=False),
|
||||||
|
sa.Column('checksum_sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('size_bytes', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('content_type', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('use_stage', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('used_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_attachment_uses_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_attachment_uses_campaign_job_id_campaign_jobs'), ondelete='SET NULL'),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_attachment_uses_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['file_asset_id'], ['file_assets.id'], name=op.f('fk_campaign_attachment_uses_file_asset_id_file_assets'), ondelete='RESTRICT'),
|
||||||
|
sa.ForeignKeyConstraint(['file_blob_id'], ['file_blobs.id'], name=op.f('fk_campaign_attachment_uses_file_blob_id_file_blobs'), ondelete='RESTRICT'),
|
||||||
|
sa.ForeignKeyConstraint(['file_version_id'], ['file_versions.id'], name=op.f('fk_campaign_attachment_uses_file_version_id_file_versions'), ondelete='RESTRICT'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_attachment_uses')),
|
||||||
|
sa.UniqueConstraint('campaign_job_id', 'file_version_id', 'filename_used', 'use_stage', name='uq_campaign_attachment_uses_job_file_stage')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_campaign_id'), 'campaign_attachment_uses', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_campaign_job_id'), 'campaign_attachment_uses', ['campaign_job_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_campaign_version_id'), 'campaign_attachment_uses', ['campaign_version_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_entry_id'), 'campaign_attachment_uses', ['entry_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_file_asset_id'), 'campaign_attachment_uses', ['file_asset_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_file_blob_id'), 'campaign_attachment_uses', ['file_blob_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_file_version_id'), 'campaign_attachment_uses', ['file_version_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_tenant_id'), 'campaign_attachment_uses', ['tenant_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_use_stage'), 'campaign_attachment_uses', ['use_stage'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_attachment_uses_used_at'), 'campaign_attachment_uses', ['used_at'], unique=False)
|
||||||
|
op.create_table('campaign_issues',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('campaign_version_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('job_id', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('severity', sa.String(length=20), nullable=False),
|
||||||
|
sa.Column('code', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('message', sa.Text(), nullable=False),
|
||||||
|
sa.Column('source', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('behavior', sa.String(length=50), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_campaign_version_id'), 'campaign_issues', ['campaign_version_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_code'), 'campaign_issues', ['code'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_job_id'), 'campaign_issues', ['job_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_severity'), 'campaign_issues', ['severity'], unique=False)
|
||||||
|
op.create_index(op.f('ix_campaign_issues_tenant_id'), 'campaign_issues', ['tenant_id'], unique=False)
|
||||||
|
op.create_table('imap_append_attempts',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('folder', sa.String(length=500), nullable=True),
|
||||||
|
sa.Column('status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('error_message', sa.Text(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_imap_append_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_imap_append_attempts'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_imap_append_attempts_job_id'), 'imap_append_attempts', ['job_id'], unique=False)
|
||||||
|
op.create_table('send_attempts',
|
||||||
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('job_id', sa.String(length=36), nullable=False),
|
||||||
|
sa.Column('attempt_number', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('status', sa.String(length=50), nullable=False),
|
||||||
|
sa.Column('claim_token', sa.String(length=36), nullable=True),
|
||||||
|
sa.Column('smtp_status_code', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('smtp_response', sa.Text(), nullable=True),
|
||||||
|
sa.Column('error_type', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('error_message', sa.Text(), nullable=True),
|
||||||
|
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_send_attempts_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_send_attempts'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_send_attempts_claim_token'), 'send_attempts', ['claim_token'], unique=False)
|
||||||
|
op.create_index(op.f('ix_send_attempts_job_id'), 'send_attempts', ['job_id'], unique=False)
|
||||||
|
op.create_index(op.f('ix_send_attempts_status'), 'send_attempts', ['status'], unique=False)
|
||||||
|
_seed_core_defaults()
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table('send_attempts')
|
||||||
|
op.drop_table('imap_append_attempts')
|
||||||
|
op.drop_table('campaign_issues')
|
||||||
|
op.drop_table('campaign_attachment_uses')
|
||||||
|
op.drop_table('campaign_jobs')
|
||||||
|
op.drop_table('file_versions')
|
||||||
|
op.drop_table('file_shares')
|
||||||
|
op.drop_table('campaign_versions')
|
||||||
|
op.drop_table('campaign_shares')
|
||||||
|
op.drop_table('audit_log')
|
||||||
|
op.drop_table('attachment_instances')
|
||||||
|
op.drop_table('mail_server_profiles')
|
||||||
|
op.drop_table('file_folders')
|
||||||
|
op.drop_table('file_assets')
|
||||||
|
op.drop_table('campaigns')
|
||||||
|
op.drop_table('access_user_role_assignments')
|
||||||
|
op.drop_table('access_user_group_memberships')
|
||||||
|
op.drop_table('access_auth_sessions')
|
||||||
|
op.drop_table('access_api_keys')
|
||||||
|
op.drop_table('admin_governance_template_assignments')
|
||||||
|
op.drop_table('access_users')
|
||||||
|
op.drop_table('access_system_role_assignments')
|
||||||
|
op.drop_table('access_group_role_assignments')
|
||||||
|
op.drop_table('file_blobs')
|
||||||
|
op.drop_table('core_system_settings')
|
||||||
|
op.drop_table('core_change_sequence_retention_floor')
|
||||||
|
op.drop_table('core_change_sequence')
|
||||||
|
op.drop_table('audit_outbox_events')
|
||||||
|
op.drop_table('attachment_blobs')
|
||||||
|
op.drop_table('admin_governance_templates')
|
||||||
|
op.drop_table('access_roles')
|
||||||
|
op.drop_table('access_groups')
|
||||||
|
op.drop_table('access_accounts')
|
||||||
|
op.drop_table('core_scopes')
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
GOVOPLAN_POSTGRES_DB=govoplan
|
|
||||||
GOVOPLAN_POSTGRES_USER=govoplan
|
|
||||||
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
|
|
||||||
GOVOPLAN_POSTGRES_PORT=55432
|
|
||||||
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
# PostgreSQL Development Profile
|
|
||||||
|
|
||||||
GovOPlaN development now defaults to PostgreSQL:
|
|
||||||
|
|
||||||
```text
|
|
||||||
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
|
||||||
```
|
|
||||||
|
|
||||||
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
|
||||||
```
|
|
||||||
|
|
||||||
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
|
|
||||||
The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if
|
|
||||||
you need a different database.
|
|
||||||
|
|
||||||
## Host PostgreSQL Setup
|
|
||||||
|
|
||||||
Create the default local role and database from a host shell:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
export GOVOPLAN_PG_DB=govoplan_dev
|
|
||||||
export GOVOPLAN_PG_USER=govoplan_dev
|
|
||||||
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
|
|
||||||
echo
|
|
||||||
|
|
||||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
|
|
||||||
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
|
|
||||||
else
|
|
||||||
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
|
|
||||||
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
|
|
||||||
|
|
||||||
touch ~/.pgpass
|
|
||||||
chmod 600 ~/.pgpass
|
|
||||||
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
|
|
||||||
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
|
|
||||||
mv ~/.pgpass.tmp ~/.pgpass
|
|
||||||
chmod 600 ~/.pgpass
|
|
||||||
```
|
|
||||||
|
|
||||||
Check access:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
|
|
||||||
-c 'select current_user, current_database();'
|
|
||||||
```
|
|
||||||
|
|
||||||
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
|
|
||||||
available through:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
flatpak-spawn --host /usr/bin/psql --version
|
|
||||||
flatpak-spawn --host /usr/bin/pg_dump --version
|
|
||||||
flatpak-spawn --host /usr/bin/pg_restore --version
|
|
||||||
```
|
|
||||||
|
|
||||||
## Run GovOPlaN Against Host PostgreSQL
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
|
||||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
|
||||||
--with-dev-data
|
|
||||||
|
|
||||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
|
||||||
```
|
|
||||||
|
|
||||||
For the full backend and WebUI launcher:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
scripts/launch-dev.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
To force the old SQLite fallback for a disposable local run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## Disposable Docker Testbed
|
|
||||||
|
|
||||||
This testbed is for migration and module-permutation checks. It is not a
|
|
||||||
production deployment profile.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
|
||||||
cp .env.example .env
|
|
||||||
docker compose --env-file .env up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
Run the integration check from the core checkout:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
set -a
|
|
||||||
. dev/postgres/.env
|
|
||||||
set +a
|
|
||||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
|
||||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
|
||||||
--reset-schema
|
|
||||||
```
|
|
||||||
|
|
||||||
`--reset-schema` drops and recreates the `public` schema before every module
|
|
||||||
set. Use it only against this disposable database.
|
|
||||||
|
|
||||||
Stop the testbed:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
|
||||||
docker compose --env-file .env down
|
|
||||||
```
|
|
||||||
|
|
||||||
Remove all test data:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose --env-file .env down -v
|
|
||||||
```
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16-alpine
|
|
||||||
container_name: govoplan-core-postgres-dev
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
|
|
||||||
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
|
|
||||||
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- CMD-SHELL
|
|
||||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 20
|
|
||||||
volumes:
|
|
||||||
- postgres-data:/var/lib/postgresql/data
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
postgres-data:
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
APP_ENV=staging
|
|
||||||
GOVOPLAN_INSTALL_PROFILE=production-like
|
|
||||||
MASTER_KEY_B64=
|
|
||||||
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
|
|
||||||
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
|
||||||
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
|
|
||||||
|
|
||||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
|
||||||
GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
|
||||||
REDIS_URL=redis://127.0.0.1:56379/0
|
|
||||||
CELERY_ENABLED=true
|
|
||||||
CELERY_QUEUES=send_email,append_sent,default
|
|
||||||
|
|
||||||
ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops
|
|
||||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
|
|
||||||
AUTH_COOKIE_SECURE=false
|
|
||||||
FILE_STORAGE_BACKEND=local
|
|
||||||
FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files
|
|
||||||
DEV_AUTO_MIGRATE_ENABLED=false
|
|
||||||
DEV_BOOTSTRAP_ENABLED=true
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
# Production-Like Development Profile
|
|
||||||
|
|
||||||
This profile runs the shared services that production depends on while keeping
|
|
||||||
API, worker, and WebUI code in the editable local repositories.
|
|
||||||
|
|
||||||
It provides:
|
|
||||||
|
|
||||||
- PostgreSQL with a persistent Docker volume
|
|
||||||
- Redis with append-only persistence
|
|
||||||
- explicit `ENABLED_MODULES`
|
|
||||||
- local durable file storage under `runtime/production-like/files`
|
|
||||||
- a Celery worker process using the same queues as the API
|
|
||||||
|
|
||||||
Start it from the core repository:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
scripts/launch-production-like-dev.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
The helper wrapper exposes repeatable lifecycle commands:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
scripts/production-like-dev.sh validate-config
|
|
||||||
scripts/production-like-dev.sh seed
|
|
||||||
scripts/production-like-dev.sh start
|
|
||||||
scripts/production-like-dev.sh stop
|
|
||||||
scripts/production-like-dev.sh reset --yes
|
|
||||||
```
|
|
||||||
|
|
||||||
The launcher uses `dev/production-like/.env` when present, otherwise
|
|
||||||
`dev/production-like/.env.example`. Copy the example when you want local port or
|
|
||||||
password changes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp dev/production-like/.env.example dev/production-like/.env
|
|
||||||
```
|
|
||||||
|
|
||||||
The API and worker use:
|
|
||||||
|
|
||||||
```text
|
|
||||||
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
|
||||||
REDIS_URL=redis://127.0.0.1:56379/0
|
|
||||||
CELERY_ENABLED=true
|
|
||||||
```
|
|
||||||
|
|
||||||
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
|
|
||||||
containers keep running by default so the next launch is fast. To stop them too:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
To remove all profile data:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
scripts/production-like-dev.sh reset --yes
|
|
||||||
```
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16-alpine
|
|
||||||
container_name: govoplan-production-like-postgres
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
|
|
||||||
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
|
|
||||||
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- CMD-SHELL
|
|
||||||
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 20
|
|
||||||
volumes:
|
|
||||||
- postgres-data:/var/lib/postgresql/data
|
|
||||||
|
|
||||||
redis:
|
|
||||||
image: redis:7-alpine
|
|
||||||
container_name: govoplan-production-like-redis
|
|
||||||
command: ["redis-server", "--appendonly", "yes"]
|
|
||||||
ports:
|
|
||||||
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 20
|
|
||||||
volumes:
|
|
||||||
- redis-data:/data
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
postgres-data:
|
|
||||||
redis-data:
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# GovOPlaN RBAC And Resource-Access Model
|
# GovOPlaN RBAC And Resource-Access Model
|
||||||
|
|
||||||
**Updated:** 2026-07-09
|
**Updated:** 2026-07-11
|
||||||
|
|
||||||
## Authorization Equation
|
## Authorization Equation
|
||||||
|
|
||||||
@@ -254,6 +254,61 @@ space/folder/file ownership:
|
|||||||
External file connections and spaces are additionally constrained by connector
|
External file connections and spaces are additionally constrained by connector
|
||||||
policy and owner/group assignment in the files module.
|
policy and owner/group assignment in the files module.
|
||||||
|
|
||||||
|
## Resource Access Explanations
|
||||||
|
|
||||||
|
The access module exposes a diagnostic endpoint for explaining why a principal
|
||||||
|
can see or operate on a concrete resource:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/v1/admin/access/resource-explanation
|
||||||
|
```
|
||||||
|
|
||||||
|
Required query values:
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `user_id` | Tenant membership to explain. The current module UIs pass the signed-in user. |
|
||||||
|
| `resource_type` | Module-owned type such as `file`, `folder`, or `campaign`. |
|
||||||
|
| `resource_id` | Stable module resource identifier. |
|
||||||
|
| `action` | Permission/action being explained, for example `files:file:read`. |
|
||||||
|
| `tenant_id` | Optional tenant override for system/admin contexts. |
|
||||||
|
|
||||||
|
The access module always contributes effective-scope provenance for the action.
|
||||||
|
Installed modules may add resource provenance by exposing a
|
||||||
|
`ResourceAccessExplanationProvider` through a capability consumed by access.
|
||||||
|
Providers should return only facts they own, using these provenance kinds:
|
||||||
|
|
||||||
|
| Kind | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `resource` | The concrete resource or an explicit not-found result. |
|
||||||
|
| `owner` | Matching user/group ownership. |
|
||||||
|
| `share` | Matching explicit user/group/tenant share. |
|
||||||
|
| `policy` | Administrative bypass or policy-derived grant. |
|
||||||
|
| `role` / `right` | Scope and role provenance from access itself. |
|
||||||
|
|
||||||
|
Files currently registers `files.access` and explains file assets plus folders.
|
||||||
|
Persisted folders use their database ID. Folder rows inferred from file paths use
|
||||||
|
a deterministic virtual ID:
|
||||||
|
|
||||||
|
```text
|
||||||
|
virtual-folder:v1:<tenant_id>:<owner_type>:<owner_id>:<base64url(normalized_path)>
|
||||||
|
```
|
||||||
|
|
||||||
|
The files provider validates virtual folder IDs before returning provenance: the
|
||||||
|
tenant and owner must match the resource ID, and at least one active file must
|
||||||
|
exist below the normalized folder path. This keeps virtual folder explanations
|
||||||
|
stable without forcing every inferred tree node to become a stored folder row.
|
||||||
|
|
||||||
|
Campaign currently registers `campaigns.access` and explains the campaign
|
||||||
|
ownership/sharing object itself. Finer-grained campaign sub-objects are tracked
|
||||||
|
separately in `govoplan-campaign#50` until their resource identifiers and access
|
||||||
|
rules are decided.
|
||||||
|
|
||||||
|
Cross-user resource explanation is a policy feature, not a module-local UI
|
||||||
|
detail. Until `govoplan-policy#6` is resolved, module UIs should default to
|
||||||
|
current-user explanation and avoid importing access-admin user-picking
|
||||||
|
components.
|
||||||
|
|
||||||
## Mail Servers
|
## Mail Servers
|
||||||
|
|
||||||
| Scope | Meaning |
|
| Scope | Meaning |
|
||||||
|
|||||||
@@ -58,18 +58,18 @@ Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.
|
|||||||
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./scripts/check-focused.sh
|
tools/checks/check-focused.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
For smaller changes, prefer the narrow command named in the relevant `AGENTS.md` file. Examples:
|
For smaller changes, prefer the narrow command named in the relevant `AGENTS.md` file. Examples:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest tests.test_module_system
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-mail
|
cd /mnt/DATA/git/govoplan-mail
|
||||||
/mnt/DATA/git/govoplan-core/.venv/bin/python -m unittest discover -s tests
|
/mnt/DATA/git/govoplan/.venv/bin/python -m unittest discover -s tests
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-core/webui
|
cd /mnt/DATA/git/govoplan-core/webui
|
||||||
PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations
|
PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run test:module-permutations
|
||||||
|
|||||||
@@ -6,23 +6,24 @@ metadata and can fail for newly disclosed advisories without a source change.
|
|||||||
|
|
||||||
## Local Workflow
|
## Local Workflow
|
||||||
|
|
||||||
Install the development audit dependency once:
|
Install the whole-product development dependencies once from the meta
|
||||||
|
repository:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Run both backend and WebUI production audits:
|
Run both backend and WebUI production audits:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
bash scripts/check-dependency-audits.sh
|
bash tools/checks/check-dependency-audits.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The script runs:
|
The script runs:
|
||||||
|
|
||||||
- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
- `tools/checks/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||||
legacy editable package metadata, deprecated framework constants, and the
|
legacy editable package metadata, deprecated framework constants, and the
|
||||||
Starlette `TestClient` deprecation smoke when test dependencies are present
|
Starlette `TestClient` deprecation smoke when test dependencies are present
|
||||||
- `python -m pip_audit --progress-spinner off`
|
- `python -m pip_audit --progress-spinner off`
|
||||||
@@ -31,11 +32,12 @@ The script runs:
|
|||||||
For fast local checks without vulnerability metadata lookups, run:
|
For fast local checks without vulnerability metadata lookups, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
CHECK_TESTCLIENT_DEPRECATIONS=1 \
|
||||||
|
bash tools/checks/check-dependency-hygiene.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
This is also part of `scripts/check-focused.sh`, so resolver drift and
|
This is also part of `govoplan/tools/checks/check-focused.sh`, so resolver drift and
|
||||||
deprecation regressions fail close to the code change that introduced them.
|
deprecation regressions fail close to the code change that introduced them.
|
||||||
|
|
||||||
Override tool paths when testing from a disposable environment:
|
Override tool paths when testing from a disposable environment:
|
||||||
@@ -43,12 +45,13 @@ Override tool paths when testing from a disposable environment:
|
|||||||
```bash
|
```bash
|
||||||
PYTHON=/tmp/govoplan-audit/bin/python \
|
PYTHON=/tmp/govoplan-audit/bin/python \
|
||||||
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
||||||
bash scripts/check-dependency-audits.sh
|
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core \
|
||||||
|
bash /mnt/DATA/git/govoplan/tools/checks/check-dependency-audits.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
## CI Workflow
|
## CI Workflow
|
||||||
|
|
||||||
`.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
`govoplan/.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||||
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
||||||
pull requests, and a weekly schedule.
|
pull requests, and a weekly schedule.
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ installer CLI/daemon for package mutation under maintenance mode.
|
|||||||
Generate a deployment-local template:
|
Generate a deployment-local template:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||||
--profile self-hosted \
|
--profile self-hosted \
|
||||||
--generate-secrets \
|
--generate-secrets \
|
||||||
@@ -54,8 +54,9 @@ PY
|
|||||||
| Setting | Default | Notes |
|
| Setting | Default | Notes |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
||||||
|
| `GOVOPLAN_MIGRATION_TRACK` | `release` | Use the release track for normal runtime and deployments. Use `dev` only for fresh/disposable databases that intentionally replay detailed development migrations. |
|
||||||
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
||||||
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `govoplan/tools/launch/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||||
|
|
||||||
Operator rule: take a database backup before applying migrations or destructive
|
Operator rule: take a database backup before applying migrations or destructive
|
||||||
module retirement. For non-SQLite databases, configure deployment-specific
|
module retirement. For non-SQLite databases, configure deployment-specific
|
||||||
@@ -68,10 +69,11 @@ supported only for tiny disposable profiles and unit-test style smoke runs.
|
|||||||
Production/staging deployments should use a managed PostgreSQL database and
|
Production/staging deployments should use a managed PostgreSQL database and
|
||||||
explicit migration commands.
|
explicit migration commands.
|
||||||
|
|
||||||
Install the server extra so the `psycopg` driver is available:
|
Install the full release profile from the meta repository so core, modules, and
|
||||||
|
the `psycopg` driver are available:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -113,30 +115,30 @@ pg_restore --clean --if-exists \
|
|||||||
```
|
```
|
||||||
|
|
||||||
For local development, create the host database described in
|
For local development, create the host database described in
|
||||||
`dev/postgres/README.md`, then run:
|
`/mnt/DATA/git/govoplan/dev/postgres/README.md`, then run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m govoplan_core.commands.init_db \
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||||
--with-dev-data
|
--with-dev-data
|
||||||
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
scripts/launch-dev.sh
|
tools/launch/launch-dev.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
For disposable local validation against a throwaway PostgreSQL instance, use
|
For disposable local validation against a throwaway PostgreSQL instance, use
|
||||||
the bundled PostgreSQL testbed:
|
the bundled PostgreSQL testbed:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
docker compose --env-file .env up -d
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
set -a
|
set -a
|
||||||
. dev/postgres/.env
|
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||||
set +a
|
set +a
|
||||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
tools/checks/postgres-integration-check.py \
|
||||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
--reset-schema
|
--reset-schema
|
||||||
```
|
```
|
||||||
@@ -218,7 +220,7 @@ configuration, not the core runtime contract. Store them in a local ignored
|
|||||||
## First Deployment Flow
|
## First Deployment Flow
|
||||||
|
|
||||||
1. Create an environment file or secret set with the runtime contract above.
|
1. Create an environment file or secret set with the runtime contract above.
|
||||||
2. Install the tagged core and module packages from `requirements-release.txt`.
|
2. Install the tagged core and module packages from meta `requirements-release.txt`.
|
||||||
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
||||||
artifact from the same release tag.
|
artifact from the same release tag.
|
||||||
4. Run database migrations with the target `DATABASE_URL`.
|
4. Run database migrations with the target `DATABASE_URL`.
|
||||||
@@ -251,22 +253,22 @@ WebUI code in the editable repositories while Docker provides PostgreSQL and
|
|||||||
Redis:
|
Redis:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
scripts/launch-production-like-dev.sh
|
tools/launch/launch-production-like-dev.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The helper wrapper provides explicit lifecycle commands:
|
The helper wrapper provides explicit lifecycle commands:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/production-like-dev.sh validate-config
|
tools/launch/production-like-dev.sh validate-config
|
||||||
scripts/production-like-dev.sh seed
|
tools/launch/production-like-dev.sh seed
|
||||||
scripts/production-like-dev.sh start
|
tools/launch/production-like-dev.sh start
|
||||||
scripts/production-like-dev.sh stop
|
tools/launch/production-like-dev.sh stop
|
||||||
scripts/production-like-dev.sh reset --yes
|
tools/launch/production-like-dev.sh reset --yes
|
||||||
```
|
```
|
||||||
|
|
||||||
The launcher uses `dev/production-like/.env` when present, otherwise the checked
|
The launcher uses `govoplan/dev/production-like/.env` when present, otherwise
|
||||||
in `.env.example`. It runs:
|
the checked in `.env.example`. It runs:
|
||||||
|
|
||||||
- PostgreSQL on `127.0.0.1:55433`
|
- PostgreSQL on `127.0.0.1:55433`
|
||||||
- Redis on `127.0.0.1:56379`
|
- Redis on `127.0.0.1:56379`
|
||||||
@@ -286,7 +288,7 @@ deployment test.
|
|||||||
To stop PostgreSQL and Redis when the launcher exits:
|
To stop PostgreSQL and Redis when the launcher exits:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 tools/launch/launch-production-like-dev.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
## Module Install/Uninstall Operations
|
## Module Install/Uninstall Operations
|
||||||
@@ -392,7 +394,7 @@ Run the rollback drill before relying on installer automation in a new
|
|||||||
environment:
|
environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./.venv/bin/python scripts/module-installer-rollback-drill.py --format json
|
/mnt/DATA/git/govoplan/tools/checks/module-installer-rollback-drill.py --format json
|
||||||
```
|
```
|
||||||
|
|
||||||
The drill uses temporary SQLite databases and simulated package commands. It
|
The drill uses temporary SQLite databases and simulated package commands. It
|
||||||
|
|||||||
@@ -133,13 +133,12 @@ Access:
|
|||||||
|
|
||||||
Tenancy:
|
Tenancy:
|
||||||
|
|
||||||
- `tenancy.tenant.created`
|
- `tenant.created`
|
||||||
- `tenancy.tenant.updated`
|
- `tenant.updated`
|
||||||
- `tenancy.tenant.suspended`
|
- `tenant.suspended`
|
||||||
- `tenancy.tenant.reactivated`
|
- `tenant.resumed`
|
||||||
- `tenancy.tenant.delete_requested`
|
- `tenant.deletion_requested`
|
||||||
- `tenancy.tenant.delete_blocked`
|
- `tenant.erasure_completed`
|
||||||
- `tenancy.tenant.deleted`
|
|
||||||
|
|
||||||
Policy:
|
Policy:
|
||||||
|
|
||||||
|
|||||||
@@ -1,319 +1,16 @@
|
|||||||
# Gitea Issues And Wiki Workflow
|
# Gitea Issues And Wiki Workflow
|
||||||
|
|
||||||
Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes.
|
The shared GovOPlaN Gitea issue, label, and wiki workflow tooling moved to the
|
||||||
|
meta repository.
|
||||||
|
|
||||||
The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state.
|
Use:
|
||||||
|
|
||||||
## Initial Setup
|
|
||||||
|
|
||||||
The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`.
|
|
||||||
|
|
||||||
The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan-core.git`). Override inference when needed:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export GITEA_URL=https://git.add-ideas.de
|
cd /mnt/DATA/git/govoplan
|
||||||
export GITEA_OWNER=add-ideas
|
tools/gitea/gitea-sync-labels.py --help
|
||||||
export GITEA_REPO=govoplan-core
|
tools/gitea/gitea-sync-wiki.py --help
|
||||||
export GITEA_TOKEN=...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens:
|
Canonical documentation:
|
||||||
|
|
||||||
```bash
|
- `/mnt/DATA/git/govoplan/docs/GITEA_ISSUES.md`
|
||||||
GITEA_TOKEN=...
|
|
||||||
# Optional if origin inference is not enough:
|
|
||||||
GITEA_URL=https://git.add-ideas.de
|
|
||||||
GITEA_OWNER=add-ideas
|
|
||||||
GITEA_REPO=govoplan-core
|
|
||||||
```
|
|
||||||
|
|
||||||
For a shared credentials file outside the target repository, pass `--env-file`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
Create a Gitea token with issue read/write access and label-management
|
|
||||||
permission for the repository. On scoped-token instances, this usually means
|
|
||||||
issue read/write and, if label writes are rejected, repository write permission
|
|
||||||
too.
|
|
||||||
|
|
||||||
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
|
|
||||||
Creating or updating organization labels requires a token with
|
|
||||||
`write:organization`. Repository label management only needs repository label
|
|
||||||
permission, but it duplicates the taxonomy into each repository.
|
|
||||||
|
|
||||||
Preview and apply labels:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./scripts/gitea-sync-labels.py
|
|
||||||
./scripts/gitea-sync-labels.py --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
Preview and apply the shared taxonomy as organization labels:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
|
|
||||||
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The import helpers resolve repository labels and organization labels. Repository
|
|
||||||
labels win when a repository defines the same name locally, but a repository does
|
|
||||||
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
|
|
||||||
`module/*`, `area/*`, `source/*`, or `codex/*` label.
|
|
||||||
|
|
||||||
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
|
||||||
|
|
||||||
## Multiple Repositories And Workspaces
|
|
||||||
|
|
||||||
The helper scripts are path-based and can run from this core checkout against any repository with a Gitea remote:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./scripts/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply
|
|
||||||
./scripts/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail
|
|
||||||
./scripts/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress
|
|
||||||
```
|
|
||||||
|
|
||||||
Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH.
|
|
||||||
|
|
||||||
When one core checkout drives another workspace, prefer `--env-file` for shared credentials instead of putting `GITEA_REPO` in the core `.env`; a repo-specific `GITEA_REPO` can accidentally override target inference.
|
|
||||||
|
|
||||||
For non-GovOPlaN projects, provide a project-specific label file and module/project label:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-sync-labels.py \
|
|
||||||
--root /path/to/project \
|
|
||||||
--labels-file /path/to/project/docs/gitea-labels.json \
|
|
||||||
--apply
|
|
||||||
|
|
||||||
./scripts/gitea-todo-import.py \
|
|
||||||
--root /path/to/project \
|
|
||||||
--module-label project/example \
|
|
||||||
--extra-label area/backend
|
|
||||||
```
|
|
||||||
|
|
||||||
If another project does not use `area/*` labels, disable area inference:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-todo-import.py \
|
|
||||||
--root /path/to/project \
|
|
||||||
--module-label project/example \
|
|
||||||
--no-area-labels
|
|
||||||
```
|
|
||||||
|
|
||||||
Install or refresh the shared issue templates in sibling or external repositories:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail
|
|
||||||
./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-<name>` checkout maps to `module/<name>`. For another workspace or repository name, pass an explicit label:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-install-workflow.py /path/to/repo --module-label module/example --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in core and run the sync script from core.
|
|
||||||
|
|
||||||
For a fully portable workflow kit, copy these files into the other project:
|
|
||||||
|
|
||||||
- `scripts/gitea_common.py`
|
|
||||||
- `scripts/gitea-sync-labels.py`
|
|
||||||
- `scripts/gitea-todo-import.py`
|
|
||||||
- `scripts/gitea-codex-note.py`
|
|
||||||
- `scripts/gitea-install-workflow.py`
|
|
||||||
- `.gitea/ISSUE_TEMPLATE/*`
|
|
||||||
- `.gitea/PULL_REQUEST_TEMPLATE.md`
|
|
||||||
- a project-specific `docs/gitea-labels.json`
|
|
||||||
|
|
||||||
Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup.
|
|
||||||
|
|
||||||
## Label Taxonomy
|
|
||||||
|
|
||||||
Use one `type/*` label:
|
|
||||||
|
|
||||||
- `type/bug`
|
|
||||||
- `type/feature`
|
|
||||||
- `type/task`
|
|
||||||
- `type/debt`
|
|
||||||
- `type/docs`
|
|
||||||
|
|
||||||
Use one `status/*` label while the issue is open:
|
|
||||||
|
|
||||||
- `status/triage`: needs ownership, priority, or acceptance criteria.
|
|
||||||
- `status/ready`: ready to implement.
|
|
||||||
- `status/in-progress`: actively being worked.
|
|
||||||
- `status/blocked`: blocked on an external dependency, credential, or decision.
|
|
||||||
- `status/needs-info`: blocked on clarification.
|
|
||||||
|
|
||||||
Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`.
|
|
||||||
|
|
||||||
Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository.
|
|
||||||
|
|
||||||
Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first.
|
|
||||||
|
|
||||||
## Moving TODOs Into Gitea
|
|
||||||
|
|
||||||
Preview inline markers:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./scripts/gitea-todo-import.py
|
|
||||||
```
|
|
||||||
|
|
||||||
Create missing issues after labels are synced:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-todo-import.py --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items.
|
|
||||||
|
|
||||||
When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference:
|
|
||||||
|
|
||||||
```python
|
|
||||||
# TODO(gitea#123): keep only if the local pointer is still useful
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers.
|
|
||||||
|
|
||||||
For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env
|
|
||||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
It scans repository and product-directory files with backlog-like names, resolves
|
|
||||||
shared labels from the organization label catalogue where available, creates only
|
|
||||||
missing fallback repository labels when needed, imports missing open work, and
|
|
||||||
deduplicates reruns by hidden fingerprint and normalized title.
|
|
||||||
|
|
||||||
## Mirroring Project Docs Into Gitea Wikis
|
|
||||||
|
|
||||||
Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env
|
|
||||||
```
|
|
||||||
|
|
||||||
Apply the wiki mirror:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
After renaming or deleting repository docs, prune previously managed wiki pages
|
|
||||||
that no longer have a source file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The default apply path uses the Gitea wiki git repository, not one REST API
|
|
||||||
request per page. It keeps a local checkout cache below
|
|
||||||
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
|
||||||
pushes that commit. This is much faster and avoids REST wiki-page timeouts.
|
|
||||||
Use `--transport api` only when the wiki git remote is unavailable.
|
|
||||||
|
|
||||||
Limit a sync to one repository or one generated page while working:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --apply
|
|
||||||
./scripts/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository
|
|
||||||
sync when the set of mirrored pages changes.
|
|
||||||
|
|
||||||
The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related.
|
|
||||||
|
|
||||||
Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages.
|
|
||||||
|
|
||||||
Use the wiki for durable context:
|
|
||||||
|
|
||||||
- project overviews and architecture
|
|
||||||
- workflows, operating notes, and setup references
|
|
||||||
- product concepts, plans, pitches, and whitepapers
|
|
||||||
- historical context that helps interpret issues
|
|
||||||
|
|
||||||
Keep active state in issues:
|
|
||||||
|
|
||||||
- open tasks, TODOs, feature requests, and bugs
|
|
||||||
- priority, blocking status, and acceptance criteria
|
|
||||||
- Codex progress updates and implementation notes
|
|
||||||
|
|
||||||
## Codex State Updates
|
|
||||||
|
|
||||||
Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes:
|
|
||||||
|
|
||||||
- scope understood
|
|
||||||
- files changed
|
|
||||||
- tests or manual checks run
|
|
||||||
- blockers or decisions needed
|
|
||||||
- follow-up issues created
|
|
||||||
|
|
||||||
Preview and post a standardized note:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/gitea-codex-note.py \
|
|
||||||
--issue 123 \
|
|
||||||
--status progress \
|
|
||||||
--summary "Implemented capability metadata fallback." \
|
|
||||||
--changed src/govoplan_core/modules/registry.py \
|
|
||||||
--test "./.venv/bin/python -m unittest tests.test_module_system"
|
|
||||||
|
|
||||||
./scripts/gitea-codex-note.py \
|
|
||||||
--issue 123 \
|
|
||||||
--status progress \
|
|
||||||
--summary "Implemented capability metadata fallback." \
|
|
||||||
--changed src/govoplan_core/modules/registry.py \
|
|
||||||
--test "./.venv/bin/python -m unittest tests.test_module_system" \
|
|
||||||
--apply
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded.
|
|
||||||
|
|
||||||
## Ownership Rules
|
|
||||||
|
|
||||||
Create the issue in the repository that owns the change:
|
|
||||||
|
|
||||||
- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components.
|
|
||||||
- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration.
|
|
||||||
- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations.
|
|
||||||
- `govoplan-files`: files-specific backend, frontend, storage, and file workflows.
|
|
||||||
- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior.
|
|
||||||
|
|
||||||
For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details.
|
|
||||||
|
|
||||||
## Cleaning Up Mirrored Sources
|
|
||||||
|
|
||||||
After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state.
|
|
||||||
|
|
||||||
Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding.
|
|
||||||
|
|
||||||
Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path.
|
|
||||||
|
|
||||||
## Docs Versus Issues
|
|
||||||
|
|
||||||
Keep durable facts in docs:
|
|
||||||
|
|
||||||
- architecture and extension points
|
|
||||||
- command references
|
|
||||||
- module boundaries
|
|
||||||
- operational conventions
|
|
||||||
|
|
||||||
Keep changing state in Gitea:
|
|
||||||
|
|
||||||
- TODOs and follow-ups
|
|
||||||
- bugs and feature requests
|
|
||||||
- blocked status
|
|
||||||
- acceptance criteria
|
|
||||||
- implementation notes from active work
|
|
||||||
|
|
||||||
If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history.
|
|
||||||
|
|||||||
@@ -458,6 +458,17 @@ routers and live module activation fail fast if two routers register the same
|
|||||||
HTTP method and path. That keeps OpenAPI output and FastAPI route order from
|
HTTP method and path. That keeps OpenAPI output and FastAPI route order from
|
||||||
silently masking a module collision.
|
silently masking a module collision.
|
||||||
|
|
||||||
|
Tenant deletion and cleanup use the registry-owned delete-veto contract. A
|
||||||
|
module that owns tenant-bound data may declare `delete_veto_providers` on its
|
||||||
|
manifest for resource types such as `tenant` or `group`. Providers receive
|
||||||
|
`(session, tenant_id, resource_id)` and should return `DeleteVetoIssue`, an
|
||||||
|
iterable of `DeleteVetoIssue`, or `None`; older exception-based providers are
|
||||||
|
still treated as blocking vetoes. Core attributes each issue to the provider
|
||||||
|
module and adds resource context before the tenancy module exposes the issues
|
||||||
|
through the deletion plan. `blocker` issues prevent destructive or retire
|
||||||
|
operations, `warning` issues explain retained data, and `info` issues document
|
||||||
|
non-blocking lifecycle facts.
|
||||||
|
|
||||||
## Database And Migrations
|
## Database And Migrations
|
||||||
|
|
||||||
Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest.
|
Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest.
|
||||||
@@ -470,12 +481,20 @@ Rules:
|
|||||||
- Keep cross-module foreign-key assumptions explicit and conservative.
|
- Keep cross-module foreign-key assumptions explicit and conservative.
|
||||||
- Register module metadata in `MigrationSpec` so core can discover it.
|
- Register module metadata in `MigrationSpec` so core can discover it.
|
||||||
- Optional module migrations may create multiple Alembic heads. Verification
|
- Optional module migrations may create multiple Alembic heads. Verification
|
||||||
should compare the database heads to the configured script heads instead of
|
should compare database heads to Alembic's resolved `heads` target instead
|
||||||
assuming one linear revision when multiple modules are enabled.
|
of assuming one linear revision when multiple modules are enabled. Owner
|
||||||
|
heads can be dependency parents and therefore may not all appear in
|
||||||
|
`alembic_version` after a full-graph upgrade.
|
||||||
- Treat migrations as release artifacts. Unreleased migrations may be squashed
|
- Treat migrations as release artifacts. Unreleased migrations may be squashed
|
||||||
or rewritten before a stable release; released revision IDs are immutable
|
or rewritten before a stable release; released revision IDs are immutable
|
||||||
once an installation may have recorded them. Each stable release records its
|
once an installation may have recorded them. Each stable release records its
|
||||||
public migration heads in `docs/migration-release-baselines.json`.
|
public migration heads in `docs/migration-release-baselines.json`.
|
||||||
|
- GovOPlaN keeps two Alembic tracks. The default `release` track loads
|
||||||
|
`versions` directories with reviewed release baselines and release-to-release
|
||||||
|
step-up migrations. The explicit `dev` track loads `dev_versions`
|
||||||
|
directories with the detailed development chain. Do not load both tracks for
|
||||||
|
one migration run, and do not switch a database between tracks unless it is a
|
||||||
|
disposable development database.
|
||||||
|
|
||||||
## Install, Uninstall, And Catalogs
|
## Install, Uninstall, And Catalogs
|
||||||
|
|
||||||
@@ -484,16 +503,42 @@ check, maintenance-mode guard, replay state, and installer request queue.
|
|||||||
Package mutation is performed by `govoplan-module-installer` outside the
|
Package mutation is performed by `govoplan-module-installer` outside the
|
||||||
FastAPI request process.
|
FastAPI request process.
|
||||||
|
|
||||||
Official catalogs can be served as static JSON from `govoplan-web`, but core
|
Official catalogs can be served as static JSON from `addideas-govoplan-website`, but core
|
||||||
does not trust the website by location alone. A catalog must pass the configured
|
does not trust the website by location alone. A catalog must pass the configured
|
||||||
signature, channel, freshness, and replay rules before a catalog entry can be
|
signature, channel, freshness, and replay rules before a catalog entry can be
|
||||||
planned. Catalog entries may declare `license_features`; core checks those
|
planned. Catalog entries may declare `license_features`; core checks those
|
||||||
against the configured offline license before adding the entry to the install
|
against the configured offline license before adding the entry to the install
|
||||||
plan.
|
plan. Catalog entries may also declare `migration_safety` as `automatic`,
|
||||||
|
`requires_review`, `forward_only`, or `destructive`; forward-only and
|
||||||
|
destructive entries require explicit operator acknowledgement in the install
|
||||||
|
plan before installer preflight allows activation. Forward-only and destructive
|
||||||
|
catalog entries must also declare a tested recovery path. Catalog update entries
|
||||||
|
can define direct-update windows with `current_version_min` and
|
||||||
|
`current_version_max_exclusive`, mark intermediate `bridge_release` targets, and
|
||||||
|
explicitly opt into reviewed downgrade or same-version package-refresh plans.
|
||||||
|
Module migration order can be declared with `migration_after` and
|
||||||
|
`migration_before` in manifests or release catalogs; installer preflight turns
|
||||||
|
that metadata, module dependencies, and named interface relationships into an
|
||||||
|
ordered migration plan.
|
||||||
|
|
||||||
|
Modules that need live-data work outside Alembic schema revisions may declare
|
||||||
|
`migration_tasks` on `MigrationSpec`. This is deliberately narrower than a
|
||||||
|
general lifecycle hook system. Each task has a stable `task_id`, one of four
|
||||||
|
phases (`pre_migration_check`, `pre_migration_prepare`,
|
||||||
|
`post_migration_backfill`, `post_migration_verify`), a short operator-facing
|
||||||
|
summary, a task version, safety metadata, and an idempotent executor. Installer
|
||||||
|
preflight blocks non-idempotent tasks, forward-only/destructive tasks without
|
||||||
|
operator acknowledgement, and installed manifest tasks that have no executor.
|
||||||
|
Catalog task metadata is surfaced before activation as pending because the
|
||||||
|
executor can only be verified after the package is installed.
|
||||||
|
|
||||||
Modules should provide:
|
Modules should provide:
|
||||||
|
|
||||||
- pinned backend and WebUI package refs for official catalog entries
|
- pinned backend and WebUI package refs for official catalog entries
|
||||||
|
- module dependency metadata for catalog target-state planning
|
||||||
|
- migration-safety metadata for catalog update planning
|
||||||
|
- migration task metadata when live-data checks, preparation, backfills, or
|
||||||
|
verification must run around Alembic
|
||||||
- compatibility metadata in the module manifest
|
- compatibility metadata in the module manifest
|
||||||
- named interface contracts in the manifest and catalog entry when the module
|
- named interface contracts in the manifest and catalog entry when the module
|
||||||
provides or consumes cross-module APIs
|
provides or consumes cross-module APIs
|
||||||
@@ -662,7 +707,7 @@ Rules:
|
|||||||
|
|
||||||
### Dependency Boundary Enforcement
|
### Dependency Boundary Enforcement
|
||||||
|
|
||||||
The repository includes `scripts/check_dependency_boundaries.py`. It enforces the current baseline:
|
The meta repository includes `tools/checks/check_dependency_boundaries.py`. It enforces the current baseline:
|
||||||
|
|
||||||
- kernel/core source may not add new direct imports of files/mail/campaign internals
|
- kernel/core source may not add new direct imports of files/mail/campaign internals
|
||||||
- access source may not import files/mail/campaign internals
|
- access source may not import files/mail/campaign internals
|
||||||
@@ -977,8 +1022,21 @@ The package install-plan API records operator intent only:
|
|||||||
also reports catalog validity, channel, signature, trust state, and the
|
also reports catalog validity, channel, signature, trust state, and the
|
||||||
configured path.
|
configured path.
|
||||||
- `POST /api/v1/admin/system/modules/install-plan/catalog/{module_id}` saves
|
- `POST /api/v1/admin/system/modules/install-plan/catalog/{module_id}` saves
|
||||||
a planned install row from a validated catalog entry. Catalog signature and
|
a planned install or update row from a validated catalog entry. Installed
|
||||||
approved-channel policy are enforced before the row is saved.
|
modules are planned as updates. Catalog signature and approved-channel policy
|
||||||
|
are enforced before the row is saved. When the selected catalog row requires
|
||||||
|
companion dependency or interface-provider updates, the endpoint adds those
|
||||||
|
rows to the plan automatically. The saved plan row can also carry a
|
||||||
|
data-safety acknowledgement used by preflight for forward-only or destructive
|
||||||
|
catalog entries.
|
||||||
|
- Install-plan preflight returns a structured `target_plan` summary so the
|
||||||
|
admin UI can show current version, target version, package refs,
|
||||||
|
migration-safety level, update-window and bridge metadata, recovery metadata,
|
||||||
|
and acknowledgement state without requiring JSON editing.
|
||||||
|
- Install-plan preflight also returns a structured `migration_plan` summary with
|
||||||
|
target enabled modules and ordered module migration steps. When the installer
|
||||||
|
runs with migration enabled, the database migration command receives that
|
||||||
|
target module set and ordered module list.
|
||||||
- `POST /api/v1/admin/system/modules/{module_id}/uninstall-plan` saves a
|
- `POST /api/v1/admin/system/modules/{module_id}/uninstall-plan` saves a
|
||||||
planned non-destructive uninstall row for an installed module after it has
|
planned non-destructive uninstall row for an installed module after it has
|
||||||
been disabled. The Python distribution name is resolved from the installed
|
been disabled. The Python distribution name is resolved from the installed
|
||||||
@@ -1158,18 +1216,18 @@ Backend verification from core:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
/mnt/DATA/git/govoplan/.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
/mnt/DATA/git/govoplan/tools/checks/check_dependency_boundaries.py
|
||||||
```
|
```
|
||||||
|
|
||||||
`scripts/check-focused.sh` runs npm with an isolated temporary npm user config
|
`govoplan/tools/checks/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||||
so developer-local npm settings do not create release-check warning noise.
|
so developer-local npm settings do not create release-check warning noise.
|
||||||
|
|
||||||
Focused module contract and permutation verification:
|
Focused module contract and permutation verification:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
bash scripts/check-module-matrix.sh
|
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-module-matrix.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
Core WebUI host verification:
|
Core WebUI host verification:
|
||||||
@@ -1183,7 +1241,7 @@ Clean generated `dist`, `.vite`, and source-tree `__pycache__` artifacts after v
|
|||||||
|
|
||||||
## Release Dependency Rules
|
## Release Dependency Rules
|
||||||
|
|
||||||
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. Core provides:
|
Local development may use editable Python installs and local WebUI `file:` dependencies so sibling module changes reload quickly. Release builds must use tagged git refs or published packages instead. The meta repository provides:
|
||||||
|
|
||||||
- `requirements-dev.txt` for local editable backend installs
|
- `requirements-dev.txt` for local editable backend installs
|
||||||
- `requirements-release.txt` for tagged backend module installs
|
- `requirements-release.txt` for tagged backend module installs
|
||||||
|
|||||||
@@ -97,6 +97,20 @@ New backend code should import policy-owned retention behavior from
|
|||||||
`govoplan-policy` or request the capability, not add new implementation logic
|
`govoplan-policy` or request the capability, not add new implementation logic
|
||||||
to core.
|
to core.
|
||||||
|
|
||||||
|
The retention API DTOs live in `govoplan_core.privacy.schemas`.
|
||||||
|
`PrivacyRetentionPolicyItem`, `PrivacyRetentionPolicyPatchItem`,
|
||||||
|
`RETENTION_POLICY_FIELD_KEYS`, and `default_allow_lower_level_limits()` are
|
||||||
|
platform contracts because admin, access compatibility, and policy routes expose
|
||||||
|
the same stable retention payload shape. The policy engine's internal
|
||||||
|
`PrivacyRetentionPolicy` and `PrivacyRetentionPolicyPatch` models stay in
|
||||||
|
`govoplan-policy`, because they carry implementation validators and merge
|
||||||
|
behavior that are not generic API contracts.
|
||||||
|
|
||||||
|
Tenant administration DTOs remain owned by `govoplan-tenancy`; access keeps
|
||||||
|
matching compatibility DTOs only for its legacy admin surface. Admin overview
|
||||||
|
responses remain module-local because the same counters are exposed from
|
||||||
|
different menu contexts and are not yet a separately versioned platform API.
|
||||||
|
|
||||||
## Frontend Contract
|
## Frontend Contract
|
||||||
|
|
||||||
Policy UIs must:
|
Policy UIs must:
|
||||||
|
|||||||
@@ -16,23 +16,23 @@ resolve modules from tagged git refs or from a package registry.
|
|||||||
Local development:
|
Local development:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Release install from a core checkout plus tagged module repositories:
|
Release install from the meta checkout plus tagged module repositories:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
`.[server]` is resolved relative to the current working directory. If you
|
`../govoplan-core[server]` is resolved relative to the meta requirements file.
|
||||||
create the virtualenv elsewhere, still run the install command from the core
|
If you create the virtualenv elsewhere, still run the install command from the
|
||||||
checkout:
|
meta checkout:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -64,9 +64,9 @@ referenced there exist, generate the committed release lockfile without
|
|||||||
touching the development package files:
|
touching the development package files:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
scripts/generate-release-lock.sh
|
tools/release/generate-release-lock.sh
|
||||||
cd webui
|
cd /mnt/DATA/git/govoplan-core/webui
|
||||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -96,25 +96,27 @@ generated in a clean release workspace from tagged git dependencies.
|
|||||||
|
|
||||||
## Release Tag Script
|
## Release Tag Script
|
||||||
|
|
||||||
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps
|
The normal release path is automated by `/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh`: it bumps
|
||||||
or accepts the target version, updates Python/WebUI/module manifest versions,
|
or accepts the target version, updates Python/WebUI/module manifest versions,
|
||||||
commits/tags/pushes the module repositories first, regenerates
|
commits/tags/pushes the module repositories first, regenerates
|
||||||
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
||||||
working tree has already been bumped, pass the current version explicitly:
|
working tree has already been bumped, pass the current version explicitly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
scripts/push-release-tag.sh --version 0.1.6
|
tools/release/push-release-tag.sh --version 0.1.6
|
||||||
```
|
```
|
||||||
|
|
||||||
`scripts/generate-release-catalog.py` reads installed/discovered
|
`/mnt/DATA/git/govoplan/tools/release/generate-release-catalog.py` reads installed/discovered
|
||||||
`ModuleManifest` objects while writing catalog entries. When a manifest is
|
`ModuleManifest` objects while writing catalog entries. When a manifest is
|
||||||
available, the catalog entry uses the manifest version, points package refs at
|
available, the catalog entry uses the manifest version, points package refs at
|
||||||
`v<manifest.version>`, and copies `provides_interfaces` /
|
`v<manifest.version>`, and copies `provides_interfaces` /
|
||||||
`requires_interfaces` from the manifest. If a manifest cannot be discovered,
|
`requires_interfaces` from the manifest. It also copies module migration order
|
||||||
the entry falls back to the release version passed with `--version` and omits
|
and `migration_tasks` metadata when present. If a manifest cannot be
|
||||||
interface metadata. This keeps the catalog aligned with independently
|
discovered, the entry falls back to the release version passed with `--version`
|
||||||
versioned module packages instead of relying on a hardcoded compatibility table.
|
and omits interface and migration-task metadata. This keeps the catalog aligned
|
||||||
|
with independently versioned module packages instead of relying on a hardcoded
|
||||||
|
compatibility table.
|
||||||
|
|
||||||
The script also includes GovOPlaN roadmap/scaffold module repositories that do
|
The script also includes GovOPlaN roadmap/scaffold module repositories that do
|
||||||
not yet have package metadata. Those repositories are committed, tagged, and
|
not yet have package metadata. Those repositories are committed, tagged, and
|
||||||
@@ -124,7 +126,6 @@ are not listed in `requirements-release.txt` or `webui/package.release.json`.
|
|||||||
|
|
||||||
Current tag-only module repositories:
|
Current tag-only module repositories:
|
||||||
|
|
||||||
- `govoplan-addresses`
|
|
||||||
- `govoplan-appointments`
|
- `govoplan-appointments`
|
||||||
- `govoplan-cases`
|
- `govoplan-cases`
|
||||||
- `govoplan-connectors`
|
- `govoplan-connectors`
|
||||||
@@ -155,8 +156,8 @@ running server may plan and validate package changes, but package mutation is
|
|||||||
performed by the separate installer daemon or an operator shell during
|
performed by the separate installer daemon or an operator shell during
|
||||||
maintenance mode.
|
maintenance mode.
|
||||||
|
|
||||||
`govoplan-web` is the public static distribution surface for official catalog
|
`addideas-govoplan-website` is the public static distribution surface for
|
||||||
resources:
|
official catalog resources:
|
||||||
|
|
||||||
- signed module package catalogs, grouped by release channel
|
- signed module package catalogs, grouped by release channel
|
||||||
- public catalog keyrings
|
- public catalog keyrings
|
||||||
@@ -205,6 +206,27 @@ Each module entry can declare:
|
|||||||
- WebUI package name and pinned install reference
|
- WebUI package name and pinned install reference
|
||||||
- display metadata and tags
|
- display metadata and tags
|
||||||
- `license_features`, the feature entitlements required to plan that install
|
- `license_features`, the feature entitlements required to plan that install
|
||||||
|
- `dependencies` and `optional_dependencies`, the module ids expected in the
|
||||||
|
target module set
|
||||||
|
- `migration_safety`, one of `automatic`, `requires_review`, `forward_only`,
|
||||||
|
or `destructive`
|
||||||
|
- `migration_notes`, operator-facing data/migration guidance for review,
|
||||||
|
forward-only, or destructive changes
|
||||||
|
- `migration_after` and `migration_before`, explicit module ids used to order
|
||||||
|
module-owned migration heads when a release needs a live-data sequencing rule
|
||||||
|
- `migration_tasks`, constrained live-data tasks that run around Alembic
|
||||||
|
migration phases. Each task declares `task_id`, `phase`, `summary`,
|
||||||
|
`task_version`, `safety`, `idempotent`, and optionally `timeout_seconds`.
|
||||||
|
The allowed phases are `pre_migration_check`, `pre_migration_prepare`,
|
||||||
|
`post_migration_backfill`, and `post_migration_verify`.
|
||||||
|
- `current_version_min` and `current_version_max_exclusive`, the installed
|
||||||
|
version window from which this catalog target may be applied directly
|
||||||
|
- `bridge_release` and `bridge_notes`, marking a target as an intermediate
|
||||||
|
compatibility release in a staged update path
|
||||||
|
- `allow_downgrade` and `allow_same_version`, explicit opt-ins for reviewed
|
||||||
|
rollback or package-refresh plans
|
||||||
|
- `recovery_tested` and `recovery_notes`, documenting the rehearsal for
|
||||||
|
forward-only or destructive data changes
|
||||||
- `provides_interfaces`, named interface contracts exported by this module
|
- `provides_interfaces`, named interface contracts exported by this module
|
||||||
- `requires_interfaces`, named interface contracts and version ranges required
|
- `requires_interfaces`, named interface contracts and version ranges required
|
||||||
by this module
|
by this module
|
||||||
@@ -264,16 +286,32 @@ source/path, source type, cache path, channel, sequence, generated/validity
|
|||||||
timestamps, signature state, trusted key id, and cache state where available.
|
timestamps, signature state, trusted key id, and cache state where available.
|
||||||
Catalog provenance changes preflight severity:
|
Catalog provenance changes preflight severity:
|
||||||
|
|
||||||
- catalog-sourced installs require a configured, valid package catalog before
|
- catalog-sourced installs and updates require a configured, valid package
|
||||||
activation
|
catalog before activation
|
||||||
- invalid, untrusted, expired, not-yet-valid, replayed, or unapproved-channel
|
- invalid, untrusted, expired, not-yet-valid, replayed, or unapproved-channel
|
||||||
catalogs block catalog-sourced installs
|
catalogs block catalog-sourced installs and updates
|
||||||
- the same catalog validation failures remain warnings for manual install
|
- the same catalog validation failures remain warnings for manual install
|
||||||
plans, so operators can still use offline or emergency package refs
|
plans, so operators can still use offline or emergency package refs
|
||||||
- valid-catalog warnings, such as intentionally unsigned local catalogs when
|
- valid-catalog warnings, such as intentionally unsigned local catalogs when
|
||||||
signature enforcement is disabled, remain warnings
|
signature enforcement is disabled, remain warnings
|
||||||
- selected catalog entries with unsatisfied non-optional named interface ranges
|
- selected catalog entries with unsatisfied non-optional named interface ranges
|
||||||
block activation before the installer runs
|
block activation before the installer runs
|
||||||
|
- selected catalog entries whose target dependencies are neither installed nor
|
||||||
|
planned block activation before the installer runs
|
||||||
|
- catalog update targets older than the installed module version block unless
|
||||||
|
the catalog entry declares `allow_downgrade: true`
|
||||||
|
- catalog update targets equal to the installed module version block unless the
|
||||||
|
catalog entry declares `allow_same_version: true`
|
||||||
|
- catalog update targets with a `current_version_min` /
|
||||||
|
`current_version_max_exclusive` window block when the installed version is
|
||||||
|
outside that window; publish and apply a bridge release instead
|
||||||
|
- catalog entries marked `forward_only` or `destructive` block activation until
|
||||||
|
the plan row has an explicit data-safety acknowledgement
|
||||||
|
- catalog entries marked `forward_only` or `destructive` also block unless the
|
||||||
|
catalog entry declares `recovery_tested: true` and either the catalog entry or
|
||||||
|
operator plan row contains recovery notes
|
||||||
|
- catalog entries marked `destructive` also require catalog migration notes or
|
||||||
|
operator notes describing the cleanup or retirement plan
|
||||||
|
|
||||||
### Update Paths
|
### Update Paths
|
||||||
|
|
||||||
@@ -283,6 +321,41 @@ catalog validation snapshot. The installer may install multiple packages into
|
|||||||
the environment before activation, then validate the discovered manifests and
|
the environment before activation, then validate the discovered manifests and
|
||||||
activate the resulting set together.
|
activate the resulting set together.
|
||||||
|
|
||||||
|
Install-plan rows support explicit `install`, `update`, and `uninstall`
|
||||||
|
actions. Catalog planning writes `update` when the module is already installed.
|
||||||
|
Preflight resolves the target set from installed manifests plus the planned
|
||||||
|
catalog entries. Unplanned catalog entries are not treated as installed. When a
|
||||||
|
catalog entry would satisfy a missing dependency or named interface, preflight
|
||||||
|
blocks activation with a companion-update issue; the admin catalog planner adds
|
||||||
|
those companion rows automatically when it can resolve them from the current
|
||||||
|
catalog. The preflight response also includes a structured `target_plan` summary
|
||||||
|
with each planned module's action, current version, catalog target version,
|
||||||
|
package refs, migration-safety level, current-version update window, bridge
|
||||||
|
metadata, recovery metadata, and acknowledgement state.
|
||||||
|
|
||||||
|
Database migrations are planned against that same target module set. When the
|
||||||
|
installer is run with `--migrate`, it calls `govoplan_core.commands.init_db`
|
||||||
|
with the target enabled modules rather than the pre-update startup module list,
|
||||||
|
so newly installed module migration directories are discovered before
|
||||||
|
activation. Preflight also returns a structured migration plan. Its step order is
|
||||||
|
derived from:
|
||||||
|
|
||||||
|
- manifest and catalog `migration_after` / `migration_before` declarations
|
||||||
|
- module dependencies and optional dependencies when both modules are in the
|
||||||
|
target plan
|
||||||
|
- named interface provider/consumer relationships when both sides are in the
|
||||||
|
target plan
|
||||||
|
|
||||||
|
Preflight blocks cycles in that ordering graph. It also blocks non-idempotent
|
||||||
|
module migration tasks, forward-only/destructive tasks without operator
|
||||||
|
acknowledgement, and installed manifest tasks that declare no executor.
|
||||||
|
Catalog-only task executors are marked as pending because they can only be
|
||||||
|
confirmed after the target package is installed. The migrator runs pre-migration
|
||||||
|
tasks, upgrades the ordered module heads first, finishes with Alembic `heads`,
|
||||||
|
and then runs post-migration tasks, so Alembic's revision graph remains
|
||||||
|
authoritative while GovOPlaN still gives operators a module-aware live-data
|
||||||
|
order.
|
||||||
|
|
||||||
This avoids circular "upgrade A first / upgrade B first" traps: named interface
|
This avoids circular "upgrade A first / upgrade B first" traps: named interface
|
||||||
requirements are solved against the target set, not against each intermediate
|
requirements are solved against the target set, not against each intermediate
|
||||||
package-install moment. If the target set cannot satisfy all non-optional
|
package-install moment. If the target set cannot satisfy all non-optional
|
||||||
@@ -303,11 +376,25 @@ Live data upgrades need an even stricter rule:
|
|||||||
must publish an intermediate compatibility release rather than a circular
|
must publish an intermediate compatibility release rather than a circular
|
||||||
update chain
|
update chain
|
||||||
|
|
||||||
|
The release catalog is the first safety gate for this. Generated catalogs mark
|
||||||
|
modules with registered migrations as `requires_review` by default. Release
|
||||||
|
authors should keep that value for ordinary reversible migrations, raise it to
|
||||||
|
`forward_only` when database rollback requires restoring a snapshot, and raise
|
||||||
|
it to `destructive` when the update removes or irreversibly rewrites persisted
|
||||||
|
data. Forward-only and destructive entries must include `recovery_tested: true`
|
||||||
|
and recovery notes after a verified restore or forward-recovery rehearsal. The
|
||||||
|
admin install-plan UI exposes the safety level and lets operators record an
|
||||||
|
explicit acknowledgement; preflight keeps acknowledged forward-only/destructive
|
||||||
|
changes visible as warnings.
|
||||||
|
|
||||||
In practice, circular dependencies are avoided by designing interfaces with
|
In practice, circular dependencies are avoided by designing interfaces with
|
||||||
compatibility windows and by publishing bridge releases. A bridge release keeps
|
compatibility windows and by publishing bridge releases. A bridge release keeps
|
||||||
the old interface while introducing the new one, allowing dependent modules to
|
the old interface while introducing the new one, allowing dependent modules to
|
||||||
move first; a later release can retire the old interface after every dependent
|
move first; a later release can retire the old interface after every dependent
|
||||||
module has a compatible target version.
|
module has a compatible target version. Use `current_version_min` and
|
||||||
|
`current_version_max_exclusive` to make those direct-update windows explicit in
|
||||||
|
the catalog, and set `bridge_release: true` on intermediate targets that exist
|
||||||
|
primarily to carry installations safely across a compatibility gap.
|
||||||
|
|
||||||
Trusted catalog keys are configured locally:
|
Trusted catalog keys are configured locally:
|
||||||
|
|
||||||
@@ -335,11 +422,11 @@ Dependency vulnerability checks are documented in
|
|||||||
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
bash scripts/check-dependency-audits.sh
|
bash tools/checks/check-dependency-audits.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same
|
The Gitea workflow in `govoplan/.gitea/workflows/dependency-audit.yml` runs the same
|
||||||
check against release dependency refs on pushes, pull requests, and a weekly
|
check against release dependency refs on pushes, pull requests, and a weekly
|
||||||
schedule.
|
schedule.
|
||||||
|
|
||||||
@@ -499,22 +586,23 @@ entitlements without changing the source license of the repositories.
|
|||||||
|
|
||||||
Production-grade distribution still needs remote registry/git artifact
|
Production-grade distribution still needs remote registry/git artifact
|
||||||
resolution before package-manager apply, a hardened catalog publishing pipeline
|
resolution before package-manager apply, a hardened catalog publishing pipeline
|
||||||
in `govoplan-web`, and automated key rotation and emergency revocation drills.
|
that writes to `addideas-govoplan-website`, and automated key rotation and
|
||||||
|
emergency revocation drills.
|
||||||
|
|
||||||
## Release Catalog Publishing
|
## Release Catalog Publishing
|
||||||
|
|
||||||
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
GovOPlaN release catalogs are published to `addideas-govoplan-website` as
|
||||||
verified by `govoplan-core` before installer plans are accepted. Private signing
|
static JSON and verified by `govoplan-core` before installer plans are
|
||||||
keys must stay outside all git repositories. Public keyrings are published with
|
accepted. Private signing keys must stay outside all git repositories. Public
|
||||||
the website.
|
keyrings are published with the website.
|
||||||
|
|
||||||
Create the first catalog signing key on the release machine:
|
Create the first catalog signing key on the release machine:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
mkdir -p "$KEY_DIR"
|
mkdir -p "$KEY_DIR"
|
||||||
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
./.venv/bin/python tools/release/generate-catalog-keypair.py \
|
||||||
--key-id release-key-1 \
|
--key-id release-key-1 \
|
||||||
--private-key "$KEY_DIR/release-key-1.pem" \
|
--private-key "$KEY_DIR/release-key-1.pem" \
|
||||||
--public-key "$KEY_DIR/release-key-1.pub" \
|
--public-key "$KEY_DIR/release-key-1.pub" \
|
||||||
@@ -524,12 +612,12 @@ mkdir -p "$KEY_DIR"
|
|||||||
Keep `release-key-1.pem` private. The generated keyring contains only public
|
Keep `release-key-1.pem` private. The generated keyring contains only public
|
||||||
material.
|
material.
|
||||||
|
|
||||||
Generate the signed catalog into `govoplan-web`:
|
Generate the signed catalog into `addideas-govoplan-website`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
scripts/publish-release-catalog.sh \
|
tools/release/publish-release-catalog.sh \
|
||||||
--version <x.y.z> \
|
--version <x.y.z> \
|
||||||
--sequence 202607071340 \
|
--sequence 202607071340 \
|
||||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
@@ -538,8 +626,8 @@ scripts/publish-release-catalog.sh \
|
|||||||
|
|
||||||
This writes:
|
This writes:
|
||||||
|
|
||||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/channels/stable.json`
|
||||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
- `/mnt/DATA/git/addideas-govoplan-website/public/catalogs/v1/keyring.json`
|
||||||
|
|
||||||
The wrapper validates the catalog with core using the generated public keyring.
|
The wrapper validates the catalog with core using the generated public keyring.
|
||||||
|
|
||||||
@@ -547,14 +635,14 @@ For normal module/core releases, first audit and record migration baselines,
|
|||||||
then tag and push the module/core repos. Finally publish the website catalog:
|
then tag and push the module/core repos. Finally publish the website catalog:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||||
```
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
scripts/publish-release-catalog.sh \
|
tools/release/publish-release-catalog.sh \
|
||||||
--version <x.y.z> \
|
--version <x.y.z> \
|
||||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
--build-web \
|
--build-web \
|
||||||
@@ -575,16 +663,16 @@ The public keyring URL is:
|
|||||||
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
||||||
```
|
```
|
||||||
|
|
||||||
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` can publish the web catalog after module and core
|
||||||
tags have been pushed. It runs the migration release audit in automatic mode:
|
tags have been pushed. It runs the migration release audit in automatic mode:
|
||||||
warning-only before the first recorded migration baseline, strict after a
|
warning-only before the first recorded migration baseline, strict after a
|
||||||
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
||||||
mode explicitly:
|
mode explicitly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
scripts/push-release-tag.sh \
|
tools/release/push-release-tag.sh \
|
||||||
--bump subversion \
|
--bump subversion \
|
||||||
--strict-migration-audit \
|
--strict-migration-audit \
|
||||||
--publish-web-catalog \
|
--publish-web-catalog \
|
||||||
@@ -596,6 +684,50 @@ Use `--catalog-signing-key` more than once during a key rotation window. The
|
|||||||
catalog will contain multiple signatures and the public keyring will include the
|
catalog will contain multiple signatures and the public keyring will include the
|
||||||
corresponding public keys.
|
corresponding public keys.
|
||||||
|
|
||||||
|
## Release Doctor
|
||||||
|
|
||||||
|
Use `/mnt/DATA/git/govoplan/tools/release/release-doctor.py` before release preparation and again before
|
||||||
|
publishing. It inspects repository state, local versions, release tags,
|
||||||
|
migration audit state, release package refs, stable catalog/keyring state, and
|
||||||
|
optionally public catalog availability. The default mode is read-only and
|
||||||
|
offline. Status output is intentionally concise:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan
|
||||||
|
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z>
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--details` when the full findings should be printed directly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python tools/release/release-doctor.py status --target-version <x.y.z> --details
|
||||||
|
```
|
||||||
|
|
||||||
|
For concise guidance, print only suggested next commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python tools/release/release-doctor.py next --target-version <x.y.z>
|
||||||
|
```
|
||||||
|
|
||||||
|
For an operator-guided session, run interactive mode. The doctor asks which
|
||||||
|
suggested command to run; mutating commands require typing `RUN` before they
|
||||||
|
execute:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python tools/release/release-doctor.py interactive --target-version <x.y.z>
|
||||||
|
```
|
||||||
|
|
||||||
|
Interactive mode starts with a compact summary and waits for an action. It can
|
||||||
|
open the full report in the configured pager, run one suggested command, repair
|
||||||
|
Git `safe.directory` dubious-ownership blocks after explicit confirmation, push
|
||||||
|
all clean repositories that are ahead of their upstream, or commit and push all
|
||||||
|
dirty repositories. The dirty-repository bulk action asks for a commit message,
|
||||||
|
skips repositories that are behind upstream, and requires typing
|
||||||
|
`COMMIT AND PUSH` before it stages, commits, and pushes.
|
||||||
|
|
||||||
|
Add `--online` when remote tag and public catalog/keyring reachability should be
|
||||||
|
checked. Add `--json` when a CI job or another tool should consume the report.
|
||||||
|
|
||||||
On a GovOPlaN installation that should consume the official stable catalog:
|
On a GovOPlaN installation that should consume the official stable catalog:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -619,7 +751,7 @@ operator-supervised package update with restart and health checks.
|
|||||||
Key rotation for published catalogs:
|
Key rotation for published catalogs:
|
||||||
|
|
||||||
1. Generate the next private key outside git.
|
1. Generate the next private key outside git.
|
||||||
2. Run `publish-release-catalog.sh` with both signing keys.
|
2. Run `/mnt/DATA/git/govoplan/tools/release/publish-release-catalog.sh` with both signing keys.
|
||||||
3. Publish the web catalog/keyring.
|
3. Publish the web catalog/keyring.
|
||||||
4. Roll the new public keyring into installations.
|
4. Roll the new public keyring into installations.
|
||||||
5. Stop signing with the old key after the supported fleet trusts the new key.
|
5. Stop signing with the old key after the supported fleet trusts the new key.
|
||||||
@@ -633,15 +765,15 @@ smoke check before tagging or publishing catalogs. Start the local testbed,
|
|||||||
then run the permutation check from the core checkout:
|
then run the permutation check from the core checkout:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
cd /mnt/DATA/git/govoplan/dev/postgres
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
docker compose --env-file .env up -d
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
set -a
|
set -a
|
||||||
. dev/postgres/.env
|
. /mnt/DATA/git/govoplan/dev/postgres/.env
|
||||||
set +a
|
set +a
|
||||||
./.venv/bin/python scripts/postgres-integration-check.py \
|
tools/checks/postgres-integration-check.py \
|
||||||
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
--reset-schema
|
--reset-schema
|
||||||
```
|
```
|
||||||
@@ -654,56 +786,94 @@ throwaway database.
|
|||||||
## Migration Baselines
|
## Migration Baselines
|
||||||
|
|
||||||
Development migrations may be small and numerous while a feature is moving.
|
Development migrations may be small and numerous while a feature is moving.
|
||||||
Before a stable release, unreleased migrations may be rewritten or squashed into
|
GovOPlaN keeps those detailed migrations on an explicit development track and
|
||||||
a release-level baseline or release-to-release upgrade migration. After a
|
publishes reviewed release shortcuts on the release track. Before a stable
|
||||||
release tag has shipped, released migration revision IDs are immutable.
|
release, unreleased development migrations may be squashed into a release-level
|
||||||
|
baseline or release-to-release upgrade migration. After a release tag has
|
||||||
|
shipped, released migration revision IDs are immutable.
|
||||||
|
|
||||||
The release policy is:
|
The release policy is:
|
||||||
|
|
||||||
- unreleased migrations may be folded before release;
|
- unreleased development migrations live in `dev_versions` and are not deleted
|
||||||
- released migrations are never rewritten or deleted;
|
when a release shortcut is added;
|
||||||
|
- released migrations live in `versions` and are never rewritten or deleted;
|
||||||
|
- future release shortcuts are additive. A new release-to-release migration
|
||||||
|
starts from the previous recorded release heads, so installations can upgrade
|
||||||
|
through sane release steps instead of replaying every development revision;
|
||||||
- each stable release records the public migration head revisions in
|
- each stable release records the public migration head revisions in
|
||||||
`docs/migration-release-baselines.json`;
|
`docs/migration-release-baselines.json`;
|
||||||
|
- `heads` records the Alembic dependency-leaf graph heads for a full graph,
|
||||||
|
while `owner_heads` records the latest migration revision per migration
|
||||||
|
owner for subset installs and review;
|
||||||
- fresh installations should apply release-level baselines/upgrades, not
|
- fresh installations should apply release-level baselines/upgrades, not
|
||||||
unreleased create-then-rename churn;
|
unreleased create-then-rename churn;
|
||||||
- release-to-release schema changes should be folded into one reviewed
|
- release-to-release schema changes should be folded into one reviewed
|
||||||
migration per migration owner where practical.
|
migration per migration owner where practical.
|
||||||
|
|
||||||
|
The default runtime track is `release`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan
|
||||||
|
GOVOPLAN_MIGRATION_TRACK=release ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `dev` only for local/disposable development databases that intentionally
|
||||||
|
need the detailed chain:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan
|
||||||
|
GOVOPLAN_MIGRATION_TRACK=dev ./.venv/bin/python -m govoplan_core.commands.init_db
|
||||||
|
```
|
||||||
|
|
||||||
|
Production, release checks, operator install flows, and the normal development
|
||||||
|
launcher should stay on the `release` track unless a fresh/disposable database
|
||||||
|
is intentionally being used to test the detailed development chain.
|
||||||
|
|
||||||
Audit the current graph during release preparation:
|
Audit the current graph during release preparation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python scripts/release-migration-audit.py
|
./.venv/bin/python tools/release/release-migration-audit.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Audit the detailed development graph separately when a squash is prepared:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python tools/release/release-migration-audit.py --track dev
|
||||||
```
|
```
|
||||||
|
|
||||||
Generate the reviewed/manual squash checklist:
|
Generate the reviewed/manual squash checklist:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./.venv/bin/python scripts/release-migration-audit.py --squash-plan
|
./.venv/bin/python tools/release/release-migration-audit.py --squash-plan
|
||||||
```
|
```
|
||||||
|
|
||||||
After the release migrations have been reviewed and the graph is final, record
|
After the release migrations have been reviewed and the graph is final, record
|
||||||
the release baseline:
|
the release baseline:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./.venv/bin/python scripts/release-migration-audit.py --record-release <x.y.z>
|
./.venv/bin/python tools/release/release-migration-audit.py --record-release <x.y.z>
|
||||||
```
|
```
|
||||||
|
|
||||||
Use strict mode to verify that the current heads are recorded:
|
Use strict mode to verify that the current heads are recorded:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./.venv/bin/python scripts/release-migration-audit.py --strict
|
./.venv/bin/python tools/release/release-migration-audit.py --strict
|
||||||
```
|
```
|
||||||
|
|
||||||
`scripts/push-release-tag.sh` runs the audit by default in automatic mode:
|
`/mnt/DATA/git/govoplan/tools/release/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||||
non-strict while no release baseline exists, strict after the first baseline is
|
non-strict while no release baseline exists, strict after the first baseline is
|
||||||
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
||||||
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
||||||
only for emergency/manual release work.
|
only for emergency/manual release work.
|
||||||
|
|
||||||
Before the first stable release, fold the current development chain into the
|
The first public baseline is v0.1.7. It intentionally adds release-track
|
||||||
first public baseline and record that baseline in
|
shortcuts for the unreleased v0.0.0 -> v0.1.7 development chains while keeping
|
||||||
`docs/migration-release-baselines.json`. The tracking issue is
|
the detailed chains on the `dev` track. No production installations existed
|
||||||
|
before that baseline, so pre-v0.1.7 development revisions are not release
|
||||||
|
upgrade targets. Future release-to-release changes must start from a recorded
|
||||||
|
release baseline and add a new release-track step-up instead of replacing prior
|
||||||
|
release shortcuts. The tracking issue is
|
||||||
`add-ideas/govoplan-core#223`.
|
`add-ideas/govoplan-core#223`.
|
||||||
|
|
||||||
## Related Operator Documents
|
## Related Operator Documents
|
||||||
@@ -719,11 +889,11 @@ first public baseline and record that baseline in
|
|||||||
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
||||||
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
||||||
calendar, and scaffold module repositories together.
|
calendar, and scaffold module repositories together.
|
||||||
- Update `requirements-release.txt` and `webui/package.release.json` when the
|
- Update meta `requirements-release.txt` and core `webui/package.release.json` when the
|
||||||
release tag changes.
|
release tag changes.
|
||||||
- Generate the committed full-product release lockfile from
|
- Generate the committed full-product release lockfile from
|
||||||
`package.release.json` with `scripts/generate-release-lock.sh`.
|
`package.release.json` with `/mnt/DATA/git/govoplan/tools/release/generate-release-lock.sh`.
|
||||||
- Run `scripts/release-migration-audit.py --strict` after recording a release
|
- Run `/mnt/DATA/git/govoplan/tools/release/release-migration-audit.py --strict` after recording a release
|
||||||
baseline.
|
baseline.
|
||||||
- Run the PostgreSQL release check against a disposable database.
|
- Run the PostgreSQL release check against a disposable database.
|
||||||
- Publish the signed catalog through the release catalog publishing flow above.
|
- Publish the signed catalog through the release catalog publishing flow above.
|
||||||
|
|||||||
15
docs/SECURITY_AUDIT.md
Normal file
15
docs/SECURITY_AUDIT.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Security Audit Toolchain
|
||||||
|
|
||||||
|
The shared GovOPlaN security audit toolbox moved to the meta repository.
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan
|
||||||
|
tools/checks/security-audit/run.sh --mode ci --scope govoplan
|
||||||
|
tools/checks/security-audit/run.sh --mode full --scope govoplan
|
||||||
|
```
|
||||||
|
|
||||||
|
Canonical documentation:
|
||||||
|
|
||||||
|
- `/mnt/DATA/git/govoplan/docs/SECURITY_AUDIT.md`
|
||||||
@@ -22,7 +22,7 @@ daemon. The API server must not run package managers from request handlers.
|
|||||||
Generate a self-hosted template:
|
Generate a self-hosted template:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
./.venv/bin/python -m govoplan_core.commands.config env-template \
|
||||||
--profile self-hosted \
|
--profile self-hosted \
|
||||||
--generate-secrets \
|
--generate-secrets \
|
||||||
@@ -49,31 +49,31 @@ catalog source is configured.
|
|||||||
Use the local production-like wrapper for repeatable rehearsal:
|
Use the local production-like wrapper for repeatable rehearsal:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
scripts/production-like-dev.sh validate-config
|
tools/launch/production-like-dev.sh validate-config
|
||||||
scripts/production-like-dev.sh seed
|
tools/launch/production-like-dev.sh seed
|
||||||
scripts/production-like-dev.sh start
|
tools/launch/production-like-dev.sh start
|
||||||
```
|
```
|
||||||
|
|
||||||
Stop Docker dependencies:
|
Stop Docker dependencies:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/production-like-dev.sh stop
|
tools/launch/production-like-dev.sh stop
|
||||||
```
|
```
|
||||||
|
|
||||||
Reset all profile data:
|
Reset all profile data:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/production-like-dev.sh reset --yes
|
tools/launch/production-like-dev.sh reset --yes
|
||||||
```
|
```
|
||||||
|
|
||||||
The start command delegates to `scripts/launch-production-like-dev.sh`, which
|
The start command delegates to `tools/launch/launch-production-like-dev.sh`, which
|
||||||
runs API, worker, and WebUI in the foreground. Stop those processes with
|
runs API, worker, and WebUI in the foreground. Stop those processes with
|
||||||
`Ctrl+C` in the launcher terminal.
|
`Ctrl+C` in the launcher terminal.
|
||||||
|
|
||||||
## Module Boundary Gate
|
## Module Boundary Gate
|
||||||
|
|
||||||
`scripts/check_dependency_boundaries.py` is part of the focused verification
|
`govoplan/tools/checks/check_dependency_boundaries.py` is part of the focused verification
|
||||||
path. It checks backend imports and WebUI package/source imports so modules do
|
path. It checks backend imports and WebUI package/source imports so modules do
|
||||||
not grow hidden runtime dependencies on each other. Feature modules should
|
not grow hidden runtime dependencies on each other. Feature modules should
|
||||||
integrate through core capabilities, backend APIs/events, route contributions,
|
integrate through core capabilities, backend APIs/events, route contributions,
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ contestability, responsibility, and traceability at the point of action.
|
|||||||
| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit |
|
| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit |
|
||||||
| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records |
|
| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records |
|
||||||
| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments |
|
| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments |
|
||||||
|
| UX-015 | Core owns the platform appearance contract. Modules must use shared CSS tokens and shared controls for theme-aware UI; they must not define independent light/dark palette systems. | Accepted | Core shell and all module WebUIs |
|
||||||
|
|
||||||
## Confirmed Implementation Decisions
|
## Confirmed Implementation Decisions
|
||||||
|
|
||||||
@@ -138,6 +139,23 @@ adaptive form, not force a linear wizard.
|
|||||||
- Wizard shells remain available for assisted setup, first-run guidance,
|
- Wizard shells remain available for assisted setup, first-run guidance,
|
||||||
imports, discovery-heavy flows, and operational preflight workflows.
|
imports, discovery-heavy flows, and operational preflight workflows.
|
||||||
|
|
||||||
|
### DUE-008: Platform Theme Contract
|
||||||
|
|
||||||
|
Decision: the WebUI shell exposes a small, stable appearance contract based on
|
||||||
|
shared CSS tokens and persisted user preference selection.
|
||||||
|
|
||||||
|
- Core applies `system`, `light`, and `dark` preferences at the document root.
|
||||||
|
- Core owns shared tokens such as `--bg`, `--bar`, `--panel`, `--surface`,
|
||||||
|
`--line`, `--line-dark`, `--text`, `--text-strong`, `--muted`, semantic
|
||||||
|
status colors, radii, shadows, and disabled-control colors.
|
||||||
|
- Modules must style new UI with these tokens and shared controls. Module-local
|
||||||
|
CSS may tune layout and spacing, but it must not introduce a separate
|
||||||
|
appearance system.
|
||||||
|
- Appearance controls live in user settings first. Tenant defaults and policy
|
||||||
|
enforcement can be added later without changing the token contract.
|
||||||
|
- Visual preview in settings is illustrative; it must reflect token families,
|
||||||
|
not become a second theme implementation.
|
||||||
|
|
||||||
## Implementation Sequence
|
## Implementation Sequence
|
||||||
|
|
||||||
| Phase | Scope | Output |
|
| Phase | Scope | Output |
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
Commands:
|
Commands:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan
|
||||||
bash scripts/check-dependency-audits.sh
|
GOVOPLAN_CORE_ROOT=/mnt/DATA/git/govoplan-core bash tools/checks/check-dependency-audits.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
Status: remediated.
|
Status: remediated.
|
||||||
@@ -45,15 +45,15 @@ Remediation applied on 2026-07-09:
|
|||||||
- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to
|
- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to
|
||||||
`pyzipper==0.4.0`
|
`pyzipper==0.4.0`
|
||||||
- upgraded the local audit environment to `pip==26.1.2`
|
- upgraded the local audit environment to `pip==26.1.2`
|
||||||
- removed the obsolete local `govoplan-module-multimailer` editable install
|
- removed the obsolete local mailer-module editable install
|
||||||
from the audit environment so the audit reflects the split module product
|
from the audit environment so the audit reflects the split module product
|
||||||
|
|
||||||
Post-remediation result:
|
Post-remediation result:
|
||||||
|
|
||||||
- `bash scripts/check-dependency-audits.sh`: passed, no known Python
|
- `bash tools/checks/check-dependency-audits.sh`: passed, no known Python
|
||||||
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
||||||
- `python -m pip check`: passed.
|
- `python -m pip check`: passed.
|
||||||
- `bash scripts/check-module-matrix.sh`: passed.
|
- `bash tools/checks/check-module-matrix.sh`: passed.
|
||||||
- `python -m unittest tests.test_api_smoke`: passed.
|
- `python -m unittest tests.test_api_smoke`: passed.
|
||||||
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
||||||
|
|
||||||
|
|||||||
@@ -1,416 +0,0 @@
|
|||||||
[
|
|
||||||
{
|
|
||||||
"name": "type/bug",
|
|
||||||
"color": "d73a4a",
|
|
||||||
"description": "A reproducible defect, regression, or incorrect behavior.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "type/feature",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "New user-visible behavior or platform capability.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "type/task",
|
|
||||||
"color": "1d76db",
|
|
||||||
"description": "Implementation, maintenance, migration, or operational work.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "type/debt",
|
|
||||||
"color": "fbca04",
|
|
||||||
"description": "Cleanup, refactoring, risk reduction, or deferred engineering work.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "type/docs",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "Documentation, process, or developer workflow work.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "priority/p0",
|
|
||||||
"color": "b60205",
|
|
||||||
"description": "Immediate stop-the-line priority.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "priority/p1",
|
|
||||||
"color": "d93f0b",
|
|
||||||
"description": "High priority for the next focused work window.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "priority/p2",
|
|
||||||
"color": "fbca04",
|
|
||||||
"description": "Normal planned priority.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "priority/p3",
|
|
||||||
"color": "c2e0c6",
|
|
||||||
"description": "Low priority or opportunistic cleanup.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "status/triage",
|
|
||||||
"color": "d4c5f9",
|
|
||||||
"description": "Needs review, ownership, priority, or acceptance criteria.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "status/ready",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "Ready for implementation.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "status/in-progress",
|
|
||||||
"color": "0052cc",
|
|
||||||
"description": "Currently being worked.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "status/blocked",
|
|
||||||
"color": "b60205",
|
|
||||||
"description": "Cannot progress without a decision, dependency, credential, or external change.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "status/needs-info",
|
|
||||||
"color": "f9d0c4",
|
|
||||||
"description": "Needs clarifying input before implementation can proceed safely.",
|
|
||||||
"exclusive": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/access",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/addresses",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "GovOPlaN Addresses module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/admin",
|
|
||||||
"color": "006b75",
|
|
||||||
"description": "GovOPlaN Admin module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/appointments",
|
|
||||||
"color": "d876e3",
|
|
||||||
"description": "GovOPlaN Appointments module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/audit",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "GovOPlaN Audit module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/calendar",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "GovOPlaN Calendar module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/campaign",
|
|
||||||
"color": "d876e3",
|
|
||||||
"description": "GovOPlaN campaign module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/cases",
|
|
||||||
"color": "1d76db",
|
|
||||||
"description": "GovOPlaN Cases module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/connectors",
|
|
||||||
"color": "c2e0c6",
|
|
||||||
"description": "GovOPlaN Connectors module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/core",
|
|
||||||
"color": "0052cc",
|
|
||||||
"description": "GovOPlaN core runner, shared primitives, shell, or extension points.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/dms",
|
|
||||||
"color": "c5def5",
|
|
||||||
"description": "GovOPlaN Dms module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/erp",
|
|
||||||
"color": "fef2c0",
|
|
||||||
"description": "GovOPlaN Erp module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/files",
|
|
||||||
"color": "006b75",
|
|
||||||
"description": "GovOPlaN files module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/fit-connect",
|
|
||||||
"color": "fbca04",
|
|
||||||
"description": "GovOPlaN Fit Connect module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/forms",
|
|
||||||
"color": "f9d0c4",
|
|
||||||
"description": "GovOPlaN Forms module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/identity-trust",
|
|
||||||
"color": "d4c5f9",
|
|
||||||
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/identity",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "GovOPlaN Identity module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/idm",
|
|
||||||
"color": "0052cc",
|
|
||||||
"description": "GovOPlaN Idm module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/ledger",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "GovOPlaN Ledger module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/mail",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "GovOPlaN mail module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/notifications",
|
|
||||||
"color": "d876e3",
|
|
||||||
"description": "GovOPlaN Notifications module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/ops",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "GovOPlaN Ops module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/organizations",
|
|
||||||
"color": "bfdadc",
|
|
||||||
"description": "GovOPlaN Organizations module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/payments",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "GovOPlaN Payments module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/policy",
|
|
||||||
"color": "d93f0b",
|
|
||||||
"description": "GovOPlaN Policy module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/portal",
|
|
||||||
"color": "1d76db",
|
|
||||||
"description": "GovOPlaN Portal module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/postbox",
|
|
||||||
"color": "d93f0b",
|
|
||||||
"description": "GovOPlaN Postbox module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/reporting",
|
|
||||||
"color": "c2e0c6",
|
|
||||||
"description": "GovOPlaN Reporting module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/search",
|
|
||||||
"color": "bfdadc",
|
|
||||||
"description": "GovOPlaN Search module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/scheduling",
|
|
||||||
"color": "d93f0b",
|
|
||||||
"description": "GovOPlaN Scheduling module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/tasks",
|
|
||||||
"color": "c5def5",
|
|
||||||
"description": "GovOPlaN Tasks module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/templates",
|
|
||||||
"color": "fef2c0",
|
|
||||||
"description": "GovOPlaN Templates module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/tenancy",
|
|
||||||
"color": "e4e669",
|
|
||||||
"description": "GovOPlaN Tenancy module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/web",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "GovOPlaN public website, product page, or publication content.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/workflow",
|
|
||||||
"color": "f9d0c4",
|
|
||||||
"description": "GovOPlaN Workflow module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/xoev",
|
|
||||||
"color": "d4c5f9",
|
|
||||||
"description": "GovOPlaN Xoev module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/xrechnung",
|
|
||||||
"color": "0052cc",
|
|
||||||
"description": "GovOPlaN Xrechnung module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "module/xta-osci",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "GovOPlaN Xta Osci module behavior or integration.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/auth",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "Authentication, sessions, access bootstrap, or login behavior.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/tenancy",
|
|
||||||
"color": "bfdadc",
|
|
||||||
"description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/rbac",
|
|
||||||
"color": "c5def5",
|
|
||||||
"description": "Permissions, roles, delegation, or authorization policy.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/governance",
|
|
||||||
"color": "fef2c0",
|
|
||||||
"description": "Governance policy, audit, privacy, retention, or compliance behavior.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/module-system",
|
|
||||||
"color": "bfe5bf",
|
|
||||||
"description": "Module discovery, manifests, capabilities, routing, or optional integrations.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/migrations",
|
|
||||||
"color": "e4e669",
|
|
||||||
"description": "Alembic migrations, schema bootstrap, or persistence evolution.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/webui",
|
|
||||||
"color": "1d76db",
|
|
||||||
"description": "Shared WebUI shell, frontend components, routing, or frontend tests.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/api",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "HTTP API contracts, routers, schemas, or API smoke behavior.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/db",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "Database sessions, models, transactions, or persistence primitives.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/devex",
|
|
||||||
"color": "bfd4f2",
|
|
||||||
"description": "Local developer workflow, scripts, tests, tooling, or release helpers.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/release",
|
|
||||||
"color": "c2e0c6",
|
|
||||||
"description": "Versioning, release locks, tags, packaging, or dependency pins.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/docs",
|
|
||||||
"color": "5319e7",
|
|
||||||
"description": "Durable documentation and project guidance.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "area/marketing",
|
|
||||||
"color": "f9d0c4",
|
|
||||||
"description": "Public website, product messaging, publication copy, or legal page content.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "source/todo-scan",
|
|
||||||
"color": "cccccc",
|
|
||||||
"description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "source/backlog-import",
|
|
||||||
"color": "bfdadc",
|
|
||||||
"description": "Imported from markdown backlog, roadmap, plan, or TODO files.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "codex/ready",
|
|
||||||
"color": "0e8a16",
|
|
||||||
"description": "Suitable for Codex to pick up with the existing issue context.",
|
|
||||||
"exclusive": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "codex/needs-human",
|
|
||||||
"color": "f9d0c4",
|
|
||||||
"description": "Needs an explicit human decision before Codex should implement.",
|
|
||||||
"exclusive": false
|
|
||||||
}
|
|
||||||
]
|
|
||||||
@@ -1,4 +1,89 @@
|
|||||||
{
|
{
|
||||||
"version": 1,
|
"releases": [
|
||||||
"releases": []
|
{
|
||||||
|
"heads": [
|
||||||
|
{
|
||||||
|
"owner": "govoplan-campaign",
|
||||||
|
"revision": "2c3d4e5f7081"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-mail",
|
||||||
|
"revision": "3d4e5f708192"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-idm",
|
||||||
|
"revision": "8f9a0b1c2d3e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-calendar",
|
||||||
|
"revision": "9e0f1a2b3c4d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-files",
|
||||||
|
"revision": "a7b8c9d0e1f3"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"owner_heads": [
|
||||||
|
{
|
||||||
|
"owner": "govoplan-access",
|
||||||
|
"revisions": [
|
||||||
|
"4a5b6c7d8e9f"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-calendar",
|
||||||
|
"revisions": [
|
||||||
|
"9e0f1a2b3c4d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-campaign",
|
||||||
|
"revisions": [
|
||||||
|
"2c3d4e5f7081"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-core",
|
||||||
|
"revisions": [
|
||||||
|
"4f2a9c8e7b6d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-files",
|
||||||
|
"revisions": [
|
||||||
|
"a7b8c9d0e1f3"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-identity",
|
||||||
|
"revisions": [
|
||||||
|
"5c6d7e8f9a10"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-idm",
|
||||||
|
"revisions": [
|
||||||
|
"8f9a0b1c2d3e"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-mail",
|
||||||
|
"revisions": [
|
||||||
|
"3d4e5f708192"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "govoplan-organizations",
|
||||||
|
"revisions": [
|
||||||
|
"6d7e8f9a0b1c"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"recorded_at": "2026-07-11T01:39:45Z",
|
||||||
|
"release": "0.1.7",
|
||||||
|
"track": "release",
|
||||||
|
"squash_policy": "reviewed-manual"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": 1
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,29 @@
|
|||||||
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||||
"webui_package": "@govoplan/files-webui",
|
"webui_package": "@govoplan/files-webui",
|
||||||
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||||
|
"migration_safety": "forward_only",
|
||||||
|
"migration_notes": "Database rollback requires restoring the pre-update snapshot.",
|
||||||
|
"migration_after": ["access"],
|
||||||
|
"migration_before": ["campaigns"],
|
||||||
|
"migration_tasks": [
|
||||||
|
{
|
||||||
|
"task_id": "backfill_spaces",
|
||||||
|
"phase": "post_migration_backfill",
|
||||||
|
"summary": "Backfill file spaces after the schema migration.",
|
||||||
|
"task_version": "1",
|
||||||
|
"safety": "forward_only",
|
||||||
|
"idempotent": true,
|
||||||
|
"timeout_seconds": 300
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"current_version_min": "0.1.0",
|
||||||
|
"current_version_max_exclusive": "0.2.0",
|
||||||
|
"bridge_release": true,
|
||||||
|
"bridge_notes": "Keeps the 0.1.x campaign attachment interface while introducing the 0.2.x contract.",
|
||||||
|
"allow_downgrade": false,
|
||||||
|
"allow_same_version": false,
|
||||||
|
"recovery_tested": true,
|
||||||
|
"recovery_notes": "Snapshot restore and forward recovery were rehearsed on the release candidate dataset.",
|
||||||
"provides_interfaces": [
|
"provides_interfaces": [
|
||||||
{
|
{
|
||||||
"name": "files.campaign_attachments",
|
"name": "files.campaign_attachments",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-core"
|
name = "govoplan-core"
|
||||||
version = "0.1.6"
|
version = "0.1.8"
|
||||||
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
-e .[server]
|
|
||||||
-e ../govoplan-tenancy
|
|
||||||
-e ../govoplan-organizations
|
|
||||||
-e ../govoplan-identity
|
|
||||||
-e ../govoplan-access
|
|
||||||
-e ../govoplan-admin
|
|
||||||
-e ../govoplan-policy
|
|
||||||
-e ../govoplan-audit
|
|
||||||
-e ../govoplan-dashboard
|
|
||||||
-e ../govoplan-files
|
|
||||||
-e ../govoplan-mail
|
|
||||||
-e ../govoplan-campaign
|
|
||||||
-e ../govoplan-calendar
|
|
||||||
-e ../govoplan-docs
|
|
||||||
-e ../govoplan-ops
|
|
||||||
httpx==0.28.1
|
|
||||||
httpx2>=2.5,<3
|
|
||||||
pip-audit>=2.9,<3
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# Release install from tagged module repositories.
|
|
||||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
|
||||||
.[server]
|
|
||||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
|
|
||||||
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
|
|
||||||
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
|
|
||||||
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
|
|
||||||
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
|
|
||||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
|
|
||||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
|
|
||||||
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
|
|
||||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
|
|
||||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
|
|
||||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
|
|
||||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
|
|
||||||
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
|
|
||||||
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
|
||||||
NODE_BIN="$(dirname "$NPM")"
|
|
||||||
|
|
||||||
if [[ ! -x "$PYTHON" ]]; then
|
|
||||||
PYTHON=python
|
|
||||||
fi
|
|
||||||
if [[ ! -x "$NPM" ]]; then
|
|
||||||
NPM=npm
|
|
||||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$ROOT"
|
|
||||||
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
|
||||||
|
|
||||||
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
|
||||||
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
|
||||||
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
|
||||||
exit 127
|
|
||||||
fi
|
|
||||||
|
|
||||||
python_status=0
|
|
||||||
npm_status=0
|
|
||||||
|
|
||||||
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
|
||||||
|
|
||||||
cd "$ROOT/webui"
|
|
||||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
|
||||||
|
|
||||||
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
|
||||||
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
|
||||||
|
|
||||||
if [[ ! -x "$PYTHON" ]]; then
|
|
||||||
PYTHON=python
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$ROOT"
|
|
||||||
|
|
||||||
"$PYTHON" - <<'PY'
|
|
||||||
import importlib.util
|
|
||||||
import sys
|
|
||||||
|
|
||||||
if importlib.util.find_spec("pip") is None:
|
|
||||||
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
|
||||||
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
|
||||||
print(" python -m venv .venv", file=sys.stderr)
|
|
||||||
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
|
||||||
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
|
||||||
raise SystemExit(127)
|
|
||||||
PY
|
|
||||||
|
|
||||||
"$PYTHON" -m pip check
|
|
||||||
|
|
||||||
"$PYTHON" - <<'PY'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.metadata
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
prefix = pathlib.Path(sys.prefix)
|
|
||||||
legacy_patterns = (
|
|
||||||
"__editable__.govoplan_module_multimailer-*.pth",
|
|
||||||
"__editable___govoplan_module_multimailer_*_finder.py",
|
|
||||||
"govoplan_module_multimailer-*.dist-info",
|
|
||||||
)
|
|
||||||
problems: list[pathlib.Path] = []
|
|
||||||
|
|
||||||
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
|
||||||
for pattern in legacy_patterns:
|
|
||||||
problems.extend(site_packages.glob(pattern))
|
|
||||||
|
|
||||||
for dist in importlib.metadata.distributions():
|
|
||||||
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
|
||||||
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
|
||||||
|
|
||||||
if problems:
|
|
||||||
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
|
||||||
for path in sorted(set(problems)):
|
|
||||||
print(f" {path}", file=sys.stderr)
|
|
||||||
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
print("No stale legacy multimailer package metadata found.")
|
|
||||||
PY
|
|
||||||
|
|
||||||
"$PYTHON" - <<'PY'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
root = pathlib.Path.cwd()
|
|
||||||
scan_roots = [
|
|
||||||
root / "src",
|
|
||||||
root / "tests",
|
|
||||||
root.parent / "govoplan-access" / "src",
|
|
||||||
root.parent / "govoplan-admin" / "src",
|
|
||||||
root.parent / "govoplan-audit" / "src",
|
|
||||||
root.parent / "govoplan-calendar" / "src",
|
|
||||||
root.parent / "govoplan-campaign" / "src",
|
|
||||||
root.parent / "govoplan-dashboard" / "src",
|
|
||||||
root.parent / "govoplan-files" / "src",
|
|
||||||
root.parent / "govoplan-identity" / "src",
|
|
||||||
root.parent / "govoplan-mail" / "src",
|
|
||||||
root.parent / "govoplan-ops" / "src",
|
|
||||||
root.parent / "govoplan-organizations" / "src",
|
|
||||||
root.parent / "govoplan-policy" / "src",
|
|
||||||
root.parent / "govoplan-tenancy" / "src",
|
|
||||||
]
|
|
||||||
deprecated_tokens = {
|
|
||||||
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
|
||||||
}
|
|
||||||
|
|
||||||
hits: list[str] = []
|
|
||||||
for scan_root in scan_roots:
|
|
||||||
if not scan_root.exists():
|
|
||||||
continue
|
|
||||||
for path in scan_root.rglob("*.py"):
|
|
||||||
try:
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
continue
|
|
||||||
for token, replacement in deprecated_tokens.items():
|
|
||||||
if token in text:
|
|
||||||
hits.append(f"{path}: deprecated {token}. {replacement}")
|
|
||||||
|
|
||||||
if hits:
|
|
||||||
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
|
||||||
print("\n".join(hits), file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
print("Deprecated framework constant scan passed.")
|
|
||||||
PY
|
|
||||||
|
|
||||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
|
||||||
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
|
||||||
0|false|False|no|No)
|
|
||||||
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
|
||||||
;;
|
|
||||||
auto)
|
|
||||||
if "$PYTHON" - <<'PY'
|
|
||||||
import importlib.util
|
|
||||||
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
|
||||||
PY
|
|
||||||
then
|
|
||||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
|
||||||
else
|
|
||||||
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
|
||||||
"$PYTHON" - <<'PY'
|
|
||||||
import warnings
|
|
||||||
|
|
||||||
from starlette.exceptions import StarletteDeprecationWarning
|
|
||||||
|
|
||||||
warnings.simplefilter("error", StarletteDeprecationWarning)
|
|
||||||
|
|
||||||
from fastapi import FastAPI
|
|
||||||
from fastapi.testclient import TestClient
|
|
||||||
|
|
||||||
app = FastAPI()
|
|
||||||
|
|
||||||
@app.get("/dependency-hygiene-ping")
|
|
||||||
def ping() -> dict[str, bool]:
|
|
||||||
return {"ok": True}
|
|
||||||
|
|
||||||
with TestClient(app) as client:
|
|
||||||
response = client.get("/dependency-hygiene-ping")
|
|
||||||
assert response.status_code == 200
|
|
||||||
assert response.json() == {"ok": True}
|
|
||||||
|
|
||||||
print("TestClient deprecation smoke passed.")
|
|
||||||
PY
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Dependency hygiene check passed."
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="/mnt/DATA/git/govoplan-core"
|
|
||||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
|
||||||
NPM="$NODE/npm"
|
|
||||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
|
||||||
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
|
||||||
|
|
||||||
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
|
||||||
|
|
||||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
|
||||||
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
|
||||||
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
|
||||||
unset npm_config_tmp NPM_CONFIG_TMP
|
|
||||||
|
|
||||||
cd "$ROOT"
|
|
||||||
|
|
||||||
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
|
||||||
|
|
||||||
./.venv/bin/python - <<'PY'
|
|
||||||
import ast
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
roots = [
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-core/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-files/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"),
|
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/tests"),
|
|
||||||
]
|
|
||||||
|
|
||||||
errors = []
|
|
||||||
count = 0
|
|
||||||
for root in roots:
|
|
||||||
if not root.exists():
|
|
||||||
continue
|
|
||||||
for path in root.rglob("*.py"):
|
|
||||||
count += 1
|
|
||||||
try:
|
|
||||||
ast.parse(path.read_text(), filename=str(path))
|
|
||||||
except SyntaxError as exc:
|
|
||||||
errors.append(f"{path}:{exc.lineno}:{exc.offset}: {exc.msg}")
|
|
||||||
|
|
||||||
if errors:
|
|
||||||
print("\n".join(errors))
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
print(f"AST syntax check passed for {count} Python files")
|
|
||||||
PY
|
|
||||||
|
|
||||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
|
||||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
|
||||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
|
||||||
./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count
|
|
||||||
|
|
||||||
cd "$ROOT/webui"
|
|
||||||
"$NPM" run test:mail-components
|
|
||||||
"$NPM" run test:module-capabilities
|
|
||||||
"$NPM" run test:module-permutations
|
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-mail/webui
|
|
||||||
"$NPM" run test:mail-ui
|
|
||||||
|
|
||||||
cd /mnt/DATA/git/govoplan-campaign/webui
|
|
||||||
"$NPM" run test:policy-ui
|
|
||||||
"$NPM" run test:template-preview
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
|
||||||
NODE_BIN="$(dirname "$NPM")"
|
|
||||||
|
|
||||||
if [[ ! -x "$PYTHON" ]]; then
|
|
||||||
PYTHON=python
|
|
||||||
fi
|
|
||||||
if [[ ! -x "$NPM" ]]; then
|
|
||||||
NPM=npm
|
|
||||||
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$ROOT"
|
|
||||||
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
|
||||||
"$PYTHON" scripts/check_dependency_boundaries.py
|
|
||||||
|
|
||||||
cd "$ROOT/webui"
|
|
||||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
|
||||||
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import ast
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
REPOS = {
|
|
||||||
"core": ROOT / "src" / "govoplan_core",
|
|
||||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
|
||||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
|
||||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
|
||||||
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
|
||||||
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
|
||||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
|
||||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
|
||||||
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
|
||||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
|
||||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
|
||||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
|
||||||
}
|
|
||||||
PREFIXES = {
|
|
||||||
"core": "govoplan_core",
|
|
||||||
"access": "govoplan_access",
|
|
||||||
"admin": "govoplan_admin",
|
|
||||||
"tenancy": "govoplan_tenancy",
|
|
||||||
"organizations": "govoplan_organizations",
|
|
||||||
"identity": "govoplan_identity",
|
|
||||||
"policy": "govoplan_policy",
|
|
||||||
"audit": "govoplan_audit",
|
|
||||||
"dashboard": "govoplan_dashboard",
|
|
||||||
"files": "govoplan_files",
|
|
||||||
"mail": "govoplan_mail",
|
|
||||||
"campaign": "govoplan_campaign",
|
|
||||||
}
|
|
||||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
|
||||||
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
|
||||||
WEBUI_REPOS = {
|
|
||||||
"core": ROOT / "webui",
|
|
||||||
"files": ROOT.parent / "govoplan-files" / "webui",
|
|
||||||
"mail": ROOT.parent / "govoplan-mail" / "webui",
|
|
||||||
"campaign": ROOT.parent / "govoplan-campaign" / "webui",
|
|
||||||
}
|
|
||||||
WEBUI_PACKAGES = {
|
|
||||||
"core": "@govoplan/core-webui",
|
|
||||||
"files": "@govoplan/files-webui",
|
|
||||||
"mail": "@govoplan/mail-webui",
|
|
||||||
"campaign": "@govoplan/campaign-webui",
|
|
||||||
}
|
|
||||||
WEBUI_IMPORT_RE = re.compile(
|
|
||||||
r"(?:from\s+|import\s*\(\s*|require\s*\(\s*|export\s+[^;]*?\s+from\s+)"
|
|
||||||
r"[\"'](?P<package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
|
||||||
r"|import\s+[\"'](?P<side_effect_package>@govoplan/[a-z0-9_-]+-webui)(?:/[A-Za-z0-9_./-]+)?[\"']"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class AllowlistedImport:
|
|
||||||
owner: str
|
|
||||||
relative_path: str
|
|
||||||
imported_owner: str
|
|
||||||
reason: str
|
|
||||||
|
|
||||||
|
|
||||||
# Transitional exceptions. This should remain empty; add entries only with a
|
|
||||||
# dated removal plan and a capability/API contract issue.
|
|
||||||
ALLOWLIST: tuple[AllowlistedImport, ...] = ()
|
|
||||||
ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST}
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Violation:
|
|
||||||
kind: str
|
|
||||||
owner: str
|
|
||||||
path: Path
|
|
||||||
lineno: int
|
|
||||||
imported: str
|
|
||||||
imported_owner: str
|
|
||||||
|
|
||||||
|
|
||||||
def _import_owner(module_name: str) -> str | None:
|
|
||||||
for owner, prefix in PREFIXES.items():
|
|
||||||
if module_name == prefix or module_name.startswith(f"{prefix}."):
|
|
||||||
return owner
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _imports(path: Path) -> list[tuple[int, str]]:
|
|
||||||
tree = ast.parse(path.read_text(), filename=str(path))
|
|
||||||
result: list[tuple[int, str]] = []
|
|
||||||
for node in ast.walk(tree):
|
|
||||||
if isinstance(node, ast.Import):
|
|
||||||
for alias in node.names:
|
|
||||||
result.append((node.lineno, alias.name))
|
|
||||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
||||||
result.append((node.lineno, node.module))
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _relative(owner: str, path: Path) -> str:
|
|
||||||
return path.relative_to(REPOS[owner]).as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def _allowed(owner: str, path: Path, imported_owner: str) -> bool:
|
|
||||||
return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS
|
|
||||||
|
|
||||||
|
|
||||||
def _violations() -> list[Violation]:
|
|
||||||
violations: list[Violation] = []
|
|
||||||
for owner, root in REPOS.items():
|
|
||||||
if not root.exists():
|
|
||||||
continue
|
|
||||||
for path in root.rglob("*.py"):
|
|
||||||
if "__pycache__" in path.parts:
|
|
||||||
continue
|
|
||||||
for lineno, imported in _imports(path):
|
|
||||||
imported_owner = _import_owner(imported)
|
|
||||||
if imported_owner is None or imported_owner == owner:
|
|
||||||
continue
|
|
||||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
|
||||||
if not _allowed(owner, path, imported_owner):
|
|
||||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
|
||||||
elif owner == "access" and imported_owner in FEATURE_OWNERS:
|
|
||||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
|
||||||
elif owner in PLATFORM_OWNERS and imported_owner == "access":
|
|
||||||
if not _allowed(owner, path, imported_owner):
|
|
||||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
|
||||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
|
||||||
if not _allowed(owner, path, imported_owner):
|
|
||||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
|
||||||
elif owner in FEATURE_OWNERS and imported_owner == "access":
|
|
||||||
violations.append(Violation("python", owner, path, lineno, imported, imported_owner))
|
|
||||||
violations.extend(_webui_violations())
|
|
||||||
return violations
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_package_owner(package_name: str) -> str | None:
|
|
||||||
for owner, package in WEBUI_PACKAGES.items():
|
|
||||||
if package_name == package:
|
|
||||||
return owner
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_source_files(root: Path) -> list[Path]:
|
|
||||||
source_root = root / "src"
|
|
||||||
if not source_root.exists():
|
|
||||||
return []
|
|
||||||
return [
|
|
||||||
path
|
|
||||||
for path in source_root.rglob("*")
|
|
||||||
if path.suffix in {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"}
|
|
||||||
and "node_modules" not in path.parts
|
|
||||||
and "dist" not in path.parts
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_dependency_fields(payload: object) -> dict[str, object]:
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
return {}
|
|
||||||
result: dict[str, object] = {}
|
|
||||||
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
|
||||||
value = payload.get(key)
|
|
||||||
if isinstance(value, dict):
|
|
||||||
result.update(value)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_package_json_violations(owner: str, root: Path) -> list[Violation]:
|
|
||||||
violations: list[Violation] = []
|
|
||||||
package_paths = [root / "package.json"]
|
|
||||||
if root.name == "webui":
|
|
||||||
package_paths.append(root.parent / "package.json")
|
|
||||||
for package_path in tuple(dict.fromkeys(package_paths)):
|
|
||||||
if not package_path.exists():
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
payload = json.loads(package_path.read_text(encoding="utf-8"))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
return [Violation("webui-package", owner, package_path, exc.lineno, "invalid package.json", owner)]
|
|
||||||
for package_name in _webui_dependency_fields(payload):
|
|
||||||
imported_owner = _webui_package_owner(str(package_name))
|
|
||||||
if imported_owner is None or imported_owner == owner:
|
|
||||||
continue
|
|
||||||
if owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
|
||||||
violations.append(Violation("webui-package", owner, package_path, 1, str(package_name), imported_owner))
|
|
||||||
return violations
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_source_violations(owner: str, root: Path) -> list[Violation]:
|
|
||||||
violations: list[Violation] = []
|
|
||||||
for path in _webui_source_files(root):
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
for match in WEBUI_IMPORT_RE.finditer(text):
|
|
||||||
package_name = match.group("package") or match.group("side_effect_package")
|
|
||||||
imported_owner = _webui_package_owner(package_name)
|
|
||||||
if imported_owner is None or imported_owner == owner:
|
|
||||||
continue
|
|
||||||
lineno = text.count("\n", 0, match.start()) + 1
|
|
||||||
if owner == "core" and imported_owner in FEATURE_OWNERS:
|
|
||||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
|
||||||
elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS:
|
|
||||||
violations.append(Violation("webui-source", owner, path, lineno, package_name, imported_owner))
|
|
||||||
return violations
|
|
||||||
|
|
||||||
|
|
||||||
def _webui_violations() -> list[Violation]:
|
|
||||||
violations: list[Violation] = []
|
|
||||||
for owner, root in WEBUI_REPOS.items():
|
|
||||||
if not root.exists():
|
|
||||||
continue
|
|
||||||
violations.extend(_webui_package_json_violations(owner, root))
|
|
||||||
violations.extend(_webui_source_violations(owner, root))
|
|
||||||
return violations
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
violations = _violations()
|
|
||||||
if not violations:
|
|
||||||
print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
print("Dependency boundary violations:")
|
|
||||||
for item in violations:
|
|
||||||
print(
|
|
||||||
f"- {item.path}:{item.lineno}: {item.owner} {item.kind} imports {item.imported} "
|
|
||||||
f"({item.imported_owner}); use kernel capability/API/event contracts instead"
|
|
||||||
)
|
|
||||||
print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
import stat
|
|
||||||
|
|
||||||
from cryptography.hazmat.primitives import serialization
|
|
||||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.")
|
|
||||||
parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.")
|
|
||||||
parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.")
|
|
||||||
parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.")
|
|
||||||
parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled"))
|
|
||||||
parser.add_argument("--force", action="store_true", help="Overwrite existing key files.")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
private_path = args.private_key.expanduser()
|
|
||||||
public_path = args.public_key.expanduser() if args.public_key else None
|
|
||||||
keyring_path = args.keyring.expanduser() if args.keyring else None
|
|
||||||
if private_path.exists() and not args.force:
|
|
||||||
raise SystemExit(f"Private key already exists: {private_path}")
|
|
||||||
if public_path is not None and public_path.exists() and not args.force:
|
|
||||||
raise SystemExit(f"Public key already exists: {public_path}")
|
|
||||||
|
|
||||||
private_key = Ed25519PrivateKey.generate()
|
|
||||||
private_bytes = private_key.private_bytes(
|
|
||||||
encoding=serialization.Encoding.PEM,
|
|
||||||
format=serialization.PrivateFormat.PKCS8,
|
|
||||||
encryption_algorithm=serialization.NoEncryption(),
|
|
||||||
)
|
|
||||||
public_bytes = private_key.public_key().public_bytes(
|
|
||||||
encoding=serialization.Encoding.Raw,
|
|
||||||
format=serialization.PublicFormat.Raw,
|
|
||||||
)
|
|
||||||
public_base64 = base64.b64encode(public_bytes).decode("ascii")
|
|
||||||
|
|
||||||
private_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
private_path.write_bytes(private_bytes)
|
|
||||||
private_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
|
||||||
|
|
||||||
if public_path is not None:
|
|
||||||
public_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
public_path.write_text(public_base64 + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
if keyring_path is not None:
|
|
||||||
keyring_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
keyring = {
|
|
||||||
"keyring_version": "1",
|
|
||||||
"purpose": "govoplan module package catalog signatures",
|
|
||||||
"generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"),
|
|
||||||
"keys": [
|
|
||||||
{
|
|
||||||
"key_id": args.key_id,
|
|
||||||
"status": args.status,
|
|
||||||
"public_key": public_base64,
|
|
||||||
"not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
}
|
|
||||||
keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
print(f"private_key={private_path}")
|
|
||||||
if public_path is not None:
|
|
||||||
print(f"public_key={public_path}")
|
|
||||||
if keyring_path is not None:
|
|
||||||
print(f"keyring={keyring_path}")
|
|
||||||
print(f"key_id={args.key_id}")
|
|
||||||
print(f"public_key_base64={public_base64}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,377 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Generate and sign a GovOPlaN module package release catalog."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import UTC, datetime, timedelta
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from cryptography.hazmat.primitives import serialization
|
|
||||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
||||||
|
|
||||||
from govoplan_core.core.modules import ModuleManifest # noqa: E402
|
|
||||||
from govoplan_core.server.registry import available_module_manifests # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class CatalogModule:
|
|
||||||
module_id: str
|
|
||||||
repo: str
|
|
||||||
python_package: str
|
|
||||||
name: str
|
|
||||||
description: str
|
|
||||||
tags: tuple[str, ...]
|
|
||||||
webui_package: str | None = None
|
|
||||||
provides_interfaces: tuple[dict[str, object], ...] = ()
|
|
||||||
requires_interfaces: tuple[dict[str, object], ...] = ()
|
|
||||||
|
|
||||||
|
|
||||||
CATALOG_MODULES = (
|
|
||||||
CatalogModule(
|
|
||||||
module_id="tenancy",
|
|
||||||
repo="govoplan-tenancy",
|
|
||||||
python_package="govoplan-tenancy",
|
|
||||||
name="Tenancy",
|
|
||||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="organizations",
|
|
||||||
repo="govoplan-organizations",
|
|
||||||
python_package="govoplan-organizations",
|
|
||||||
name="Organizations",
|
|
||||||
description="Organization units, functions, and account-held function assignments.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="identity",
|
|
||||||
repo="govoplan-identity",
|
|
||||||
python_package="govoplan-identity",
|
|
||||||
name="Identity",
|
|
||||||
description="Canonical identities and links between identities and platform accounts.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="access",
|
|
||||||
repo="govoplan-access",
|
|
||||||
python_package="govoplan-access",
|
|
||||||
name="Access",
|
|
||||||
description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
webui_package="@govoplan/access-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="admin",
|
|
||||||
repo="govoplan-admin",
|
|
||||||
python_package="govoplan-admin",
|
|
||||||
name="Admin",
|
|
||||||
description="System settings, governance templates, module management, and admin shell contributions.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
webui_package="@govoplan/admin-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="policy",
|
|
||||||
repo="govoplan-policy",
|
|
||||||
python_package="govoplan-policy",
|
|
||||||
name="Policy",
|
|
||||||
description="Policy and governance capability module.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="audit",
|
|
||||||
repo="govoplan-audit",
|
|
||||||
python_package="govoplan-audit",
|
|
||||||
name="Audit",
|
|
||||||
description="Audit-log storage and audit administration routes.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="dashboard",
|
|
||||||
repo="govoplan-dashboard",
|
|
||||||
python_package="govoplan-dashboard",
|
|
||||||
name="Dashboard",
|
|
||||||
description="Configurable user home assembled from module-provided dashboard widgets.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
webui_package="@govoplan/dashboard-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="files",
|
|
||||||
repo="govoplan-files",
|
|
||||||
python_package="govoplan-files",
|
|
||||||
name="Files",
|
|
||||||
description="Managed file spaces and campaign attachment integration.",
|
|
||||||
tags=("official", "service-module"),
|
|
||||||
webui_package="@govoplan/files-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="mail",
|
|
||||||
repo="govoplan-mail",
|
|
||||||
python_package="govoplan-mail",
|
|
||||||
name="Mail",
|
|
||||||
description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.",
|
|
||||||
tags=("official", "service-module"),
|
|
||||||
webui_package="@govoplan/mail-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="campaigns",
|
|
||||||
repo="govoplan-campaign",
|
|
||||||
python_package="govoplan-campaign",
|
|
||||||
name="Campaigns",
|
|
||||||
description="Campaign authoring, validation, queueing, delivery control, and reports.",
|
|
||||||
tags=("official", "business-module"),
|
|
||||||
webui_package="@govoplan/campaign-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="calendar",
|
|
||||||
repo="govoplan-calendar",
|
|
||||||
python_package="govoplan-calendar",
|
|
||||||
name="Calendar",
|
|
||||||
description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.",
|
|
||||||
tags=("official", "service-module"),
|
|
||||||
webui_package="@govoplan/calendar-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="docs",
|
|
||||||
repo="govoplan-docs",
|
|
||||||
python_package="govoplan-docs",
|
|
||||||
name="Docs",
|
|
||||||
description="Configured-system documentation and evidence-aware help surfaces.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
webui_package="@govoplan/docs-webui",
|
|
||||||
),
|
|
||||||
CatalogModule(
|
|
||||||
module_id="ops",
|
|
||||||
repo="govoplan-ops",
|
|
||||||
python_package="govoplan-ops",
|
|
||||||
name="Ops",
|
|
||||||
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
|
||||||
tags=("official", "platform-module"),
|
|
||||||
webui_package="@govoplan/ops-webui",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.")
|
|
||||||
parser.add_argument("--channel", default="stable")
|
|
||||||
parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.")
|
|
||||||
parser.add_argument("--expires-days", type=int, default=90)
|
|
||||||
parser.add_argument("--catalog-output", type=Path, required=True)
|
|
||||||
parser.add_argument("--keyring-output", type=Path)
|
|
||||||
parser.add_argument(
|
|
||||||
"--catalog-signing-key",
|
|
||||||
action="append",
|
|
||||||
default=[],
|
|
||||||
metavar="KEY_ID=PRIVATE_KEY",
|
|
||||||
help="Ed25519 private key used to sign the catalog; may be repeated for rotation.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de")
|
|
||||||
parser.add_argument("--repository-base", default=GITEA_BASE)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
version = args.version.removeprefix("v")
|
|
||||||
tag = f"v{version}"
|
|
||||||
generated_at = datetime.now(tz=UTC)
|
|
||||||
sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M"))
|
|
||||||
expires_at = generated_at + timedelta(days=args.expires_days)
|
|
||||||
signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key]
|
|
||||||
|
|
||||||
catalog = _catalog_payload(
|
|
||||||
version=version,
|
|
||||||
tag=tag,
|
|
||||||
channel=args.channel,
|
|
||||||
sequence=sequence,
|
|
||||||
generated_at=generated_at,
|
|
||||||
expires_at=expires_at,
|
|
||||||
repository_base=args.repository_base.rstrip("/"),
|
|
||||||
public_base_url=args.public_base_url.rstrip("/"),
|
|
||||||
)
|
|
||||||
if signing_keys:
|
|
||||||
catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys]
|
|
||||||
|
|
||||||
output = args.catalog_output.expanduser()
|
|
||||||
output.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
if args.keyring_output is not None:
|
|
||||||
keyring_output = args.keyring_output.expanduser()
|
|
||||||
keyring_output.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
keyring_output.write_text(
|
|
||||||
json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"catalog={output}")
|
|
||||||
if args.keyring_output is not None:
|
|
||||||
print(f"keyring={args.keyring_output.expanduser()}")
|
|
||||||
print(f"channel={args.channel}")
|
|
||||||
print(f"sequence={sequence}")
|
|
||||||
print(f"version={version}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def _catalog_payload(
|
|
||||||
*,
|
|
||||||
version: str,
|
|
||||||
tag: str,
|
|
||||||
channel: str,
|
|
||||||
sequence: int,
|
|
||||||
generated_at: datetime,
|
|
||||||
expires_at: datetime,
|
|
||||||
repository_base: str,
|
|
||||||
public_base_url: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
manifests = _discovered_catalog_manifests()
|
|
||||||
modules: list[dict[str, Any]] = []
|
|
||||||
for module in CATALOG_MODULES:
|
|
||||||
manifest = manifests.get(module.module_id)
|
|
||||||
module_version = manifest.version if manifest is not None else version
|
|
||||||
module_tag = f"v{module_version.removeprefix('v')}"
|
|
||||||
entry: dict[str, Any] = {
|
|
||||||
"module_id": module.module_id,
|
|
||||||
"name": module.name,
|
|
||||||
"description": module.description,
|
|
||||||
"version": module_version,
|
|
||||||
"action": "install",
|
|
||||||
"python_package": module.python_package,
|
|
||||||
"python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{module_tag}",
|
|
||||||
"license_features": [f"module.{module.module_id}"],
|
|
||||||
"tags": list(module.tags),
|
|
||||||
}
|
|
||||||
if module.webui_package:
|
|
||||||
entry["webui_package"] = module.webui_package
|
|
||||||
entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{module_tag}"
|
|
||||||
manifest_interfaces = _manifest_interface_metadata(manifest)
|
|
||||||
entry.update(manifest_interfaces)
|
|
||||||
if module.provides_interfaces:
|
|
||||||
entry["provides_interfaces"] = [dict(item) for item in module.provides_interfaces]
|
|
||||||
if module.requires_interfaces:
|
|
||||||
entry["requires_interfaces"] = [dict(item) for item in module.requires_interfaces]
|
|
||||||
modules.append(entry)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"catalog_version": "1",
|
|
||||||
"channel": channel,
|
|
||||||
"sequence": sequence,
|
|
||||||
"generated_at": _json_datetime(generated_at),
|
|
||||||
"expires_at": _json_datetime(expires_at),
|
|
||||||
"release": {
|
|
||||||
"version": version,
|
|
||||||
"tag": tag,
|
|
||||||
"catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json",
|
|
||||||
"keyring_url": f"{public_base_url}/catalogs/v1/keyring.json",
|
|
||||||
},
|
|
||||||
"core_release": {
|
|
||||||
"name": "GovOPlaN Core",
|
|
||||||
"version": version,
|
|
||||||
"python_package": "govoplan-core",
|
|
||||||
"python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}",
|
|
||||||
"webui_package": "@govoplan/core-webui",
|
|
||||||
"webui_ref": f"{repository_base}/govoplan-core.git#{tag}",
|
|
||||||
},
|
|
||||||
"modules": modules,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _discovered_catalog_manifests() -> dict[str, ModuleManifest]:
|
|
||||||
try:
|
|
||||||
return available_module_manifests(ignore_load_errors=True)
|
|
||||||
except Exception:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def _manifest_interface_metadata(manifest: ModuleManifest | None) -> dict[str, object]:
|
|
||||||
if manifest is None:
|
|
||||||
return {}
|
|
||||||
payload: dict[str, object] = {}
|
|
||||||
if manifest.provides_interfaces:
|
|
||||||
payload["provides_interfaces"] = [
|
|
||||||
{"name": item.name, "version": item.version}
|
|
||||||
for item in manifest.provides_interfaces
|
|
||||||
]
|
|
||||||
if manifest.requires_interfaces:
|
|
||||||
requirements: list[dict[str, object]] = []
|
|
||||||
for item in manifest.requires_interfaces:
|
|
||||||
requirement: dict[str, object] = {
|
|
||||||
"name": item.name,
|
|
||||||
"optional": item.optional,
|
|
||||||
}
|
|
||||||
if item.version_min is not None:
|
|
||||||
requirement["version_min"] = item.version_min
|
|
||||||
if item.version_max_exclusive is not None:
|
|
||||||
requirement["version_max_exclusive"] = item.version_max_exclusive
|
|
||||||
requirements.append(requirement)
|
|
||||||
payload["requires_interfaces"] = requirements
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]:
|
|
||||||
key_id, separator, path_text = value.partition("=")
|
|
||||||
if not separator or not key_id.strip() or not path_text.strip():
|
|
||||||
raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem")
|
|
||||||
path = Path(path_text).expanduser()
|
|
||||||
private_key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
|
||||||
if not isinstance(private_key, Ed25519PrivateKey):
|
|
||||||
raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}")
|
|
||||||
return key_id.strip(), private_key
|
|
||||||
|
|
||||||
|
|
||||||
def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]:
|
|
||||||
signature_payload = dict(payload)
|
|
||||||
signature_payload.pop("signature", None)
|
|
||||||
signature_payload.pop("signatures", None)
|
|
||||||
signature = private_key.sign(_canonical_bytes(signature_payload))
|
|
||||||
return {
|
|
||||||
"algorithm": "ed25519",
|
|
||||||
"key_id": key_id,
|
|
||||||
"value": base64.b64encode(signature).decode("ascii"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"keyring_version": "1",
|
|
||||||
"purpose": "govoplan module package catalog signatures",
|
|
||||||
"generated_at": _json_datetime(generated_at),
|
|
||||||
"keys": [
|
|
||||||
{
|
|
||||||
"key_id": key_id,
|
|
||||||
"status": "active",
|
|
||||||
"public_key": _public_key_base64(private_key),
|
|
||||||
"not_before": generated_at.date().isoformat() + "T00:00:00Z",
|
|
||||||
}
|
|
||||||
for key_id, private_key in signing_keys
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _public_key_base64(private_key: Ed25519PrivateKey) -> str:
|
|
||||||
public_bytes = private_key.public_key().public_bytes(
|
|
||||||
encoding=serialization.Encoding.Raw,
|
|
||||||
format=serialization.PublicFormat.Raw,
|
|
||||||
)
|
|
||||||
return base64.b64encode(public_bytes).decode("ascii")
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical_bytes(payload: object) -> bytes:
|
|
||||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _json_datetime(value: datetime) -> str:
|
|
||||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'USAGE'
|
|
||||||
Usage:
|
|
||||||
scripts/generate-release-lock.sh [options]
|
|
||||||
|
|
||||||
Generates webui/package-lock.release.json from webui/package.release.json in a
|
|
||||||
temporary workspace. The normal development package.json and package-lock.json
|
|
||||||
are left untouched.
|
|
||||||
|
|
||||||
Run this after the module git tags referenced by package.release.json exist and
|
|
||||||
are reachable, and before tagging the core release commit that should contain
|
|
||||||
the regenerated release lockfile.
|
|
||||||
|
|
||||||
Options:
|
|
||||||
--npm <path> npm executable to use.
|
|
||||||
-h, --help Show this help.
|
|
||||||
USAGE
|
|
||||||
}
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
echo "error: $*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
WEBUI="$ROOT/webui"
|
|
||||||
NPM_BIN="${NPM:-}"
|
|
||||||
|
|
||||||
if [[ -z "$NPM_BIN" ]]; then
|
|
||||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
|
||||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
|
||||||
else
|
|
||||||
NPM_BIN="npm"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case "$1" in
|
|
||||||
--npm)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
NPM_BIN="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown argument: $1" >&2
|
|
||||||
usage >&2
|
|
||||||
exit 2
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
[[ -f "$WEBUI/package.release.json" ]] || fail "missing $WEBUI/package.release.json"
|
|
||||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm executable not found: $NPM_BIN"
|
|
||||||
|
|
||||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/govoplan-release-lock.XXXXXXXX")"
|
|
||||||
cleanup() {
|
|
||||||
rm -rf "$TMP_DIR"
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
cp "$WEBUI/package.release.json" "$TMP_DIR/package.json"
|
|
||||||
if [[ -f "$WEBUI/package-lock.release.json" ]]; then
|
|
||||||
cp "$WEBUI/package-lock.release.json" "$TMP_DIR/package-lock.json"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Generating release lockfile from $WEBUI/package.release.json"
|
|
||||||
echo "Temporary workspace: $TMP_DIR"
|
|
||||||
(
|
|
||||||
cd "$TMP_DIR"
|
|
||||||
PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" install --package-lock-only --ignore-scripts
|
|
||||||
)
|
|
||||||
|
|
||||||
cp "$TMP_DIR/package-lock.json" "$WEBUI/package-lock.release.json"
|
|
||||||
echo "Updated $WEBUI/package-lock.release.json"
|
|
||||||
@@ -1,580 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Import markdown backlog/planning files into Gitea issues."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import dataclasses
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from typing import Any, Iterable
|
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
|
||||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
|
||||||
|
|
||||||
REPO_ROOTS = {
|
|
||||||
"core": pathlib.Path("/mnt/DATA/git/govoplan-core"),
|
|
||||||
"access": pathlib.Path("/mnt/DATA/git/govoplan-access"),
|
|
||||||
"mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"),
|
|
||||||
"files": pathlib.Path("/mnt/DATA/git/govoplan-files"),
|
|
||||||
"campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"),
|
|
||||||
"web": pathlib.Path("/mnt/DATA/git/govoplan-web"),
|
|
||||||
}
|
|
||||||
|
|
||||||
MODULE_LABELS = {
|
|
||||||
"core": "module/core",
|
|
||||||
"access": "module/access",
|
|
||||||
"mail": "module/mail",
|
|
||||||
"files": "module/files",
|
|
||||||
"campaign": "module/campaign",
|
|
||||||
"web": "module/web",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class Candidate:
|
|
||||||
repo_key: str
|
|
||||||
title: str
|
|
||||||
body: str
|
|
||||||
labels: tuple[str, ...]
|
|
||||||
fingerprint: str
|
|
||||||
source: str
|
|
||||||
line: int
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
|
||||||
class RepoState:
|
|
||||||
target: Any
|
|
||||||
client: GiteaClient
|
|
||||||
label_ids: dict[str, int]
|
|
||||||
fingerprints: set[str]
|
|
||||||
titles: set[str]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
||||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
|
||||||
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
|
||||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
|
||||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
load_dotenv(args.env_file)
|
|
||||||
candidates = list(build_candidates(args))
|
|
||||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
|
||||||
|
|
||||||
print(f"Prepared {len(candidates)} backlog issue candidate(s).")
|
|
||||||
for repo_key, grouped in group_candidates(candidates).items():
|
|
||||||
print(f"{repo_key}: {len(grouped)} candidate(s)")
|
|
||||||
|
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
||||||
if not token:
|
|
||||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
|
||||||
print_preview(candidates)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates}))
|
|
||||||
missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])]
|
|
||||||
skipped = len(candidates) - len(missing)
|
|
||||||
|
|
||||||
print(f"Skipped {skipped} existing issue candidate(s).")
|
|
||||||
print(f"Missing {len(missing)} issue candidate(s).")
|
|
||||||
print_preview(missing)
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
created_by_repo: dict[str, list[str]] = {}
|
|
||||||
for candidate in missing:
|
|
||||||
state = states[candidate.repo_key]
|
|
||||||
validate_labels(candidate, state)
|
|
||||||
issue = state.client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
|
||||||
body={
|
|
||||||
"title": candidate.title,
|
|
||||||
"body": candidate.body,
|
|
||||||
"labels": [state.label_ids[label] for label in candidate.labels],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
number = issue.get("number") or issue.get("index")
|
|
||||||
created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}")
|
|
||||||
state.fingerprints.add(candidate.fingerprint)
|
|
||||||
state.titles.add(normalize_title(candidate.title))
|
|
||||||
|
|
||||||
print("Created issues:")
|
|
||||||
for repo_key, titles in created_by_repo.items():
|
|
||||||
print(f"{repo_key}:")
|
|
||||||
for title in titles:
|
|
||||||
print(f" {title}")
|
|
||||||
if not created_by_repo:
|
|
||||||
print(" none")
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
|
||||||
if args.product_backlog.exists():
|
|
||||||
yield from parse_product_backlog(args.product_backlog)
|
|
||||||
if args.access_plan and args.access_plan.exists():
|
|
||||||
yield from parse_access_plan(args.access_plan)
|
|
||||||
if args.web_todo.exists():
|
|
||||||
yield from parse_simple_todo(args.web_todo)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]:
|
|
||||||
headings: list[tuple[int, str]] = []
|
|
||||||
status = ""
|
|
||||||
lines = path.read_text(encoding="utf-8").splitlines()
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
line_number = index + 1
|
|
||||||
heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line)
|
|
||||||
if heading_match:
|
|
||||||
level = len(heading_match.group(1))
|
|
||||||
title = heading_match.group(2).strip()
|
|
||||||
if level == 2 and title == "Completed Work Archive":
|
|
||||||
break
|
|
||||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
|
||||||
headings.append((level, title))
|
|
||||||
status = ""
|
|
||||||
continue
|
|
||||||
|
|
||||||
status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line)
|
|
||||||
if status_match:
|
|
||||||
status = status_match.group(1).strip()
|
|
||||||
continue
|
|
||||||
|
|
||||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
|
||||||
if not item_match:
|
|
||||||
continue
|
|
||||||
|
|
||||||
item = collect_continued_item(lines, index, item_match.group(1))
|
|
||||||
section = section_title(headings)
|
|
||||||
repo_key, labels = classify_product_item(item, section)
|
|
||||||
prefix = type_prefix(labels)
|
|
||||||
title = format_title(prefix, item)
|
|
||||||
source = str(path)
|
|
||||||
body = body_for_item(
|
|
||||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
|
||||||
source=source,
|
|
||||||
line=line_number,
|
|
||||||
section=section,
|
|
||||||
status=status,
|
|
||||||
item=item,
|
|
||||||
note="Imported from the consolidated GovOPlaN product backlog.",
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key=repo_key,
|
|
||||||
title=title,
|
|
||||||
body=body,
|
|
||||||
labels=tuple(sorted(set(labels))),
|
|
||||||
fingerprint=make_fingerprint(source, line_number, repo_key, item),
|
|
||||||
source=source,
|
|
||||||
line=line_number,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]:
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
lines = text.splitlines()
|
|
||||||
yield from access_stage_candidates(path, lines)
|
|
||||||
yield from access_immediate_candidates(path, lines)
|
|
||||||
yield from access_decision_candidates(path, lines)
|
|
||||||
|
|
||||||
|
|
||||||
def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
||||||
index = 0
|
|
||||||
while index < len(lines):
|
|
||||||
match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index])
|
|
||||||
if not match:
|
|
||||||
index += 1
|
|
||||||
continue
|
|
||||||
stage = match.group(1)
|
|
||||||
name = match.group(2).strip()
|
|
||||||
start_line = index + 1
|
|
||||||
index += 1
|
|
||||||
block: list[str] = []
|
|
||||||
while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]):
|
|
||||||
block.append(lines[index])
|
|
||||||
index += 1
|
|
||||||
item = f"Access extraction Stage {stage}: {name}"
|
|
||||||
fingerprint = make_fingerprint(str(path), start_line, "core", item)
|
|
||||||
body = "\n".join(
|
|
||||||
[
|
|
||||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
"Imported from the GovOPlaN access extraction plan.",
|
|
||||||
"",
|
|
||||||
f"- Source: `{path}`",
|
|
||||||
f"- Line: `{start_line}`",
|
|
||||||
f"- Section: `Stage {stage}: {name}`",
|
|
||||||
"",
|
|
||||||
"Plan excerpt:",
|
|
||||||
"",
|
|
||||||
"```markdown",
|
|
||||||
f"### Stage {stage}: {name}",
|
|
||||||
*block,
|
|
||||||
"```",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key="core",
|
|
||||||
title=format_title("[Task]", item),
|
|
||||||
body=body,
|
|
||||||
labels=labels("core", "type/task", "area/module-system", "module/access"),
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=start_line,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
||||||
in_section = False
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
line_number = index + 1
|
|
||||||
if line == "## Immediate Backlog":
|
|
||||||
in_section = True
|
|
||||||
continue
|
|
||||||
if in_section and line.startswith("## "):
|
|
||||||
break
|
|
||||||
if not in_section:
|
|
||||||
continue
|
|
||||||
item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line)
|
|
||||||
if not item_match:
|
|
||||||
continue
|
|
||||||
item = collect_continued_item(lines, index, item_match.group(1))
|
|
||||||
title_item = access_title_item(item)
|
|
||||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
|
||||||
body = body_for_item(
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
section="Immediate Backlog",
|
|
||||||
status="open",
|
|
||||||
item=item,
|
|
||||||
note="Imported from the GovOPlaN access extraction plan immediate backlog.",
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key="core",
|
|
||||||
title=format_title("[Task]", title_item),
|
|
||||||
body=body,
|
|
||||||
labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"),
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]:
|
|
||||||
in_section = False
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
line_number = index + 1
|
|
||||||
if line == "## Open Decisions":
|
|
||||||
in_section = True
|
|
||||||
continue
|
|
||||||
if in_section and line.startswith("## "):
|
|
||||||
break
|
|
||||||
if not in_section:
|
|
||||||
continue
|
|
||||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
|
||||||
if not item_match:
|
|
||||||
continue
|
|
||||||
item = collect_continued_item(lines, index, item_match.group(1))
|
|
||||||
fingerprint = make_fingerprint(str(path), line_number, "core", item)
|
|
||||||
body = body_for_item(
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
section="Open Decisions",
|
|
||||||
status="decision needed",
|
|
||||||
item=item,
|
|
||||||
note="Imported from the GovOPlaN access extraction plan open decisions.",
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key="core",
|
|
||||||
title=format_title("[Task]", f"Decide: {item}"),
|
|
||||||
body=body,
|
|
||||||
labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"),
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]:
|
|
||||||
lines = path.read_text(encoding="utf-8").splitlines()
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
line_number = index + 1
|
|
||||||
item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line)
|
|
||||||
if not item_match:
|
|
||||||
continue
|
|
||||||
item = collect_continued_item(lines, index, item_match.group(1))
|
|
||||||
fingerprint = make_fingerprint(str(path), line_number, "web", item)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key="web",
|
|
||||||
title=format_title("[Task]", item),
|
|
||||||
body=body_for_item(
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
section="TODO",
|
|
||||||
status="open",
|
|
||||||
item=item,
|
|
||||||
note="Imported from the GovOPlaN website TODO file.",
|
|
||||||
),
|
|
||||||
labels=labels("web", "type/task", "area/marketing"),
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(path),
|
|
||||||
line=line_number,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]:
|
|
||||||
text = f"{section} {item}".lower()
|
|
||||||
repo_key = "core"
|
|
||||||
area = "area/devex"
|
|
||||||
issue_type = "type/task"
|
|
||||||
|
|
||||||
if "dsar search/export/delete" in text:
|
|
||||||
return "core", labels("core", "type/feature", "area/governance")
|
|
||||||
if "fully stream zip uploads" in text:
|
|
||||||
return "files", labels("files", "type/task", "area/api")
|
|
||||||
if "installer packaging approach" in text:
|
|
||||||
return "core", labels("core", "type/task", "area/devex")
|
|
||||||
if "profile reselection/revalidation" in text:
|
|
||||||
return "campaign", labels("campaign", "type/task", "area/api", "module/mail")
|
|
||||||
if "module boundaries" in text:
|
|
||||||
return "core", labels("core", issue_type, "area/module-system")
|
|
||||||
if "external storage connectors" in text:
|
|
||||||
repo_key = "files"
|
|
||||||
area = "area/api"
|
|
||||||
elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")):
|
|
||||||
repo_key = "campaign"
|
|
||||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
|
||||||
elif "controlled sending" in text:
|
|
||||||
repo_key = "campaign"
|
|
||||||
area = "area/api"
|
|
||||||
elif "mail profiles" in text:
|
|
||||||
repo_key = "mail"
|
|
||||||
area = "area/api"
|
|
||||||
|
|
||||||
if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")):
|
|
||||||
repo_key = "mail"
|
|
||||||
area = "area/api"
|
|
||||||
if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")):
|
|
||||||
repo_key = "files"
|
|
||||||
area = "area/api"
|
|
||||||
if "external storage connectors" not in text and any(
|
|
||||||
word in text
|
|
||||||
for word in (
|
|
||||||
"campaign",
|
|
||||||
"recipient",
|
|
||||||
"attachment",
|
|
||||||
"zip",
|
|
||||||
"evidence",
|
|
||||||
"wizard",
|
|
||||||
"send",
|
|
||||||
"archive",
|
|
||||||
"message preview",
|
|
||||||
"address book",
|
|
||||||
"carddav",
|
|
||||||
"mailing lists",
|
|
||||||
)
|
|
||||||
):
|
|
||||||
repo_key = "campaign"
|
|
||||||
area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api"
|
|
||||||
if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")):
|
|
||||||
repo_key = "core"
|
|
||||||
area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance"
|
|
||||||
if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")):
|
|
||||||
repo_key = "core"
|
|
||||||
area = "area/module-system"
|
|
||||||
if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")):
|
|
||||||
repo_key = "core"
|
|
||||||
area = "area/release"
|
|
||||||
if repo_key == "core" and not any(
|
|
||||||
phrase in text
|
|
||||||
for phrase in (
|
|
||||||
"external storage connectors",
|
|
||||||
"controlled sending",
|
|
||||||
"report, evidence",
|
|
||||||
"recipient import",
|
|
||||||
"campaign and attachment",
|
|
||||||
)
|
|
||||||
) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")):
|
|
||||||
repo_key = "core"
|
|
||||||
area = "area/devex"
|
|
||||||
if any(word in text for word in ("documentation", "handbook", "docs", "runbook")):
|
|
||||||
area = "area/docs"
|
|
||||||
if "ui lists often start at index 2" in text:
|
|
||||||
issue_type = "type/bug"
|
|
||||||
area = "area/webui"
|
|
||||||
repo_key = "core"
|
|
||||||
if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")):
|
|
||||||
issue_type = "type/feature"
|
|
||||||
if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")):
|
|
||||||
issue_type = "type/debt"
|
|
||||||
|
|
||||||
return repo_key, labels(repo_key, issue_type, area)
|
|
||||||
|
|
||||||
|
|
||||||
def collect_continued_item(lines: list[str], index: int, initial: str) -> str:
|
|
||||||
parts = [initial.strip()]
|
|
||||||
next_index = index + 1
|
|
||||||
while next_index < len(lines):
|
|
||||||
line = lines[next_index]
|
|
||||||
if not line.startswith((" ", "\t")):
|
|
||||||
break
|
|
||||||
stripped = line.strip()
|
|
||||||
if not stripped:
|
|
||||||
break
|
|
||||||
if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"):
|
|
||||||
break
|
|
||||||
parts.append(stripped)
|
|
||||||
next_index += 1
|
|
||||||
return " ".join(parts)
|
|
||||||
|
|
||||||
|
|
||||||
def labels(repo_key: str, *extra: str) -> tuple[str, ...]:
|
|
||||||
base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]}
|
|
||||||
base.update(extra)
|
|
||||||
if "status/needs-info" in base:
|
|
||||||
base.discard("status/triage")
|
|
||||||
return tuple(sorted(base))
|
|
||||||
|
|
||||||
|
|
||||||
def type_prefix(label_set: Iterable[str]) -> str:
|
|
||||||
labels_set = set(label_set)
|
|
||||||
if "type/bug" in labels_set:
|
|
||||||
return "[Bug]"
|
|
||||||
if "type/feature" in labels_set:
|
|
||||||
return "[Feature]"
|
|
||||||
if "type/debt" in labels_set:
|
|
||||||
return "[Debt]"
|
|
||||||
if "type/docs" in labels_set:
|
|
||||||
return "[Docs]"
|
|
||||||
return "[Task]"
|
|
||||||
|
|
||||||
|
|
||||||
def format_title(prefix: str, item: str) -> str:
|
|
||||||
clean = re.sub(r"\s+", " ", item).strip().rstrip(".")
|
|
||||||
title = f"{prefix} {clean}"
|
|
||||||
if len(title) <= 180:
|
|
||||||
return title
|
|
||||||
return f"{title[:177].rstrip()}..."
|
|
||||||
|
|
||||||
|
|
||||||
def access_title_item(item: str) -> str:
|
|
||||||
lowered = item.lower()
|
|
||||||
if "compatibility wrappers" in lowered:
|
|
||||||
return "Wire core auth compatibility through access capabilities"
|
|
||||||
return item
|
|
||||||
|
|
||||||
|
|
||||||
def body_for_item(
|
|
||||||
*,
|
|
||||||
fingerprint: str,
|
|
||||||
source: str,
|
|
||||||
line: int,
|
|
||||||
section: str,
|
|
||||||
status: str,
|
|
||||||
item: str,
|
|
||||||
note: str,
|
|
||||||
) -> str:
|
|
||||||
lines = [
|
|
||||||
f"<!-- codex-backlog-fingerprint:{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
note,
|
|
||||||
"",
|
|
||||||
f"- Source: `{source}`",
|
|
||||||
f"- Line: `{line}`",
|
|
||||||
f"- Section: `{section}`",
|
|
||||||
]
|
|
||||||
if status:
|
|
||||||
lines.append(f"- Source status: `{status}`")
|
|
||||||
lines.extend(
|
|
||||||
[
|
|
||||||
"",
|
|
||||||
"Imported backlog item:",
|
|
||||||
"",
|
|
||||||
"```markdown",
|
|
||||||
f"- [ ] {item}",
|
|
||||||
"```",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
|
||||||
return " > ".join(title for _, title in headings)
|
|
||||||
|
|
||||||
|
|
||||||
def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str:
|
|
||||||
stable = "\n".join([source, str(line), repo_key, item])
|
|
||||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
|
||||||
|
|
||||||
|
|
||||||
def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]:
|
|
||||||
grouped: dict[str, list[Candidate]] = {}
|
|
||||||
for candidate in candidates:
|
|
||||||
grouped.setdefault(candidate.repo_key, []).append(candidate)
|
|
||||||
return grouped
|
|
||||||
|
|
||||||
|
|
||||||
def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
|
||||||
states: dict[str, RepoState] = {}
|
|
||||||
for repo_key in repo_keys:
|
|
||||||
root = repo_root(REPO_ROOTS[repo_key])
|
|
||||||
target = infer_target(root)
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
|
||||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
|
||||||
fingerprints: set[str] = set()
|
|
||||||
titles: set[str] = set()
|
|
||||||
for issue in issues:
|
|
||||||
body = str(issue.get("body") or "")
|
|
||||||
if issue.get("state") == "closed" and "Moved to `" in body:
|
|
||||||
continue
|
|
||||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
|
||||||
fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body))
|
|
||||||
states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles)
|
|
||||||
return states
|
|
||||||
|
|
||||||
|
|
||||||
def already_imported(candidate: Candidate, state: RepoState) -> bool:
|
|
||||||
return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_title(title: str) -> str:
|
|
||||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def validate_labels(candidate: Candidate, state: RepoState) -> None:
|
|
||||||
missing = sorted(label for label in candidate.labels if label not in state.label_ids)
|
|
||||||
if missing:
|
|
||||||
joined = ", ".join(missing)
|
|
||||||
raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}")
|
|
||||||
|
|
||||||
|
|
||||||
def print_preview(candidates: list[Candidate], limit: int = 160) -> None:
|
|
||||||
for repo_key, grouped in group_candidates(candidates).items():
|
|
||||||
print(f"{repo_key}:")
|
|
||||||
for candidate in grouped[:limit]:
|
|
||||||
print(f" {candidate.title}")
|
|
||||||
if len(grouped) > limit:
|
|
||||||
print(f" ... {len(grouped) - limit} more")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Post a standardized Codex state update comment to a Gitea issue."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
|
||||||
|
|
||||||
|
|
||||||
STATUS_LABELS = {
|
|
||||||
"started": "status/in-progress",
|
|
||||||
"progress": "status/in-progress",
|
|
||||||
"blocked": "status/blocked",
|
|
||||||
"needs-info": "status/needs-info",
|
|
||||||
"ready": "status/ready",
|
|
||||||
"done": "",
|
|
||||||
"note": "",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
|
||||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
||||||
parser.add_argument("--issue", type=int, required=True, help="Gitea issue number")
|
|
||||||
parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note")
|
|
||||||
parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated")
|
|
||||||
parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated")
|
|
||||||
parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated")
|
|
||||||
parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated")
|
|
||||||
parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append")
|
|
||||||
parser.add_argument("--close", action="store_true", help="close the issue after posting the note")
|
|
||||||
parser.add_argument("--apply", action="store_true", help="post the comment and optional state update")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
root = repo_root(args.root)
|
|
||||||
load_dotenv(args.env_file or root / ".env")
|
|
||||||
target = infer_target(root, args.remote)
|
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
||||||
body = build_body(args)
|
|
||||||
|
|
||||||
print(f"Target: {target.display}")
|
|
||||||
print(f"Issue: #{args.issue}")
|
|
||||||
print(body)
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"),
|
|
||||||
body={"body": body},
|
|
||||||
)
|
|
||||||
if args.close:
|
|
||||||
client.request_json(
|
|
||||||
"PATCH",
|
|
||||||
repo_path(target.owner, target.repo, f"/issues/{args.issue}"),
|
|
||||||
body={"state": "closed"},
|
|
||||||
)
|
|
||||||
print("Posted Codex note.")
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def build_body(args: argparse.Namespace) -> str:
|
|
||||||
lines = [f"## Codex State: {args.status}", ""]
|
|
||||||
append_section(lines, "Summary", args.summary)
|
|
||||||
append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed])
|
|
||||||
append_section(lines, "Verification", [f"`{item}`" for item in args.test])
|
|
||||||
append_section(lines, "Next / Blocked", args.next)
|
|
||||||
if STATUS_LABELS[args.status]:
|
|
||||||
lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"])
|
|
||||||
if args.body_file:
|
|
||||||
lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()])
|
|
||||||
return "\n".join(lines).rstrip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def append_section(lines: list[str], title: str, items: list[str]) -> None:
|
|
||||||
if not items:
|
|
||||||
return
|
|
||||||
lines.extend([f"### {title}", ""])
|
|
||||||
lines.extend(f"- {item}" for item in items)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,597 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Import backlog-like files from git.add-ideas.de repositories into Gitea."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import csv
|
|
||||||
import dataclasses
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from typing import Any, Iterable
|
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
|
||||||
|
|
||||||
|
|
||||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
|
||||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
|
||||||
EXCLUDED_FILE_PATTERNS = (
|
|
||||||
"/.git/",
|
|
||||||
"/.venv/",
|
|
||||||
"/venv/",
|
|
||||||
"/site-packages/",
|
|
||||||
"/node_modules/",
|
|
||||||
"/__pycache__/",
|
|
||||||
"/dist/",
|
|
||||||
"/build/",
|
|
||||||
"/.cache/",
|
|
||||||
"/.module-test-build/",
|
|
||||||
)
|
|
||||||
BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE)
|
|
||||||
ACTION_RE = re.compile(
|
|
||||||
r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|"
|
|
||||||
r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|"
|
|
||||||
r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
SKIP_SECTION_RE = re.compile(
|
|
||||||
r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|"
|
|
||||||
r"available now|done|archive)",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
ACTIVE_SECTION_RE = re.compile(
|
|
||||||
r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|"
|
|
||||||
r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
GENERIC_LABELS = [
|
|
||||||
("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."),
|
|
||||||
("type/feature", "0e8a16", "New user-visible behavior or product capability."),
|
|
||||||
("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."),
|
|
||||||
("type/docs", "5319e7", "Documentation, process, or developer workflow work."),
|
|
||||||
("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."),
|
|
||||||
("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."),
|
|
||||||
("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."),
|
|
||||||
("priority/p0", "b60205", "Immediate stop-the-line priority."),
|
|
||||||
("priority/p1", "d93f0b", "High priority for the next focused work window."),
|
|
||||||
("priority/p2", "fbca04", "Normal planned priority."),
|
|
||||||
("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class RepoInfo:
|
|
||||||
root: pathlib.Path
|
|
||||||
remote: str
|
|
||||||
owner: str
|
|
||||||
repo: str
|
|
||||||
|
|
||||||
@property
|
|
||||||
def key(self) -> str:
|
|
||||||
return self.root.name
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class SourceFile:
|
|
||||||
repo_key: str
|
|
||||||
path: pathlib.Path
|
|
||||||
source_kind: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class Candidate:
|
|
||||||
repo_key: str
|
|
||||||
title: str
|
|
||||||
body: str
|
|
||||||
labels: tuple[str, ...]
|
|
||||||
priority: str
|
|
||||||
milestone: str
|
|
||||||
fingerprint: str
|
|
||||||
source: str
|
|
||||||
line: int
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
|
||||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
|
||||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
|
||||||
parser.add_argument("--apply", action="store_true")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
load_dotenv(args.env_file)
|
|
||||||
repos = discover_hosted_repos(args.git_root)
|
|
||||||
sources = discover_sources(repos, args.products_root)
|
|
||||||
candidates = list(build_candidates(sources))
|
|
||||||
candidates = dedupe_candidates(candidates)
|
|
||||||
candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title))
|
|
||||||
|
|
||||||
print(f"Hosted repositories: {len(repos)}")
|
|
||||||
print(f"Backlog-like source files: {len(sources)}")
|
|
||||||
print(f"Issue candidates: {len(candidates)}")
|
|
||||||
for repo_key, count in grouped_counts(candidates).items():
|
|
||||||
print(f" {repo_key}: {count}")
|
|
||||||
|
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
||||||
if not token:
|
|
||||||
print_preview(candidates)
|
|
||||||
print("Dry run without GITEA_TOKEN: duplicate comparison skipped.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
missing_by_repo: dict[str, list[Candidate]] = {}
|
|
||||||
skipped = 0
|
|
||||||
states: dict[str, RepoState] = {}
|
|
||||||
for repo_key in sorted({candidate.repo_key for candidate in candidates}):
|
|
||||||
states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply)
|
|
||||||
for candidate in candidates:
|
|
||||||
state = states[candidate.repo_key]
|
|
||||||
if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles:
|
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
missing_by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
|
||||||
|
|
||||||
missing_total = sum(len(items) for items in missing_by_repo.values())
|
|
||||||
print(f"Skipped existing: {skipped}")
|
|
||||||
print(f"Missing candidates: {missing_total}")
|
|
||||||
print_preview([candidate for items in missing_by_repo.values() for candidate in items])
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
print("Dry run only. Re-run with --apply to create missing issues.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
for repo_key, repo_candidates in missing_by_repo.items():
|
|
||||||
state = states[repo_key]
|
|
||||||
for candidate in repo_candidates:
|
|
||||||
label_ids = [state.label_ids[label] for label in candidate.labels]
|
|
||||||
if candidate.priority not in candidate.labels:
|
|
||||||
label_ids.append(state.label_ids[candidate.priority])
|
|
||||||
milestone_id = ensure_milestone(state, candidate.milestone)
|
|
||||||
created = state.client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(state.target.owner, state.target.repo, "/issues"),
|
|
||||||
body={
|
|
||||||
"title": candidate.title,
|
|
||||||
"body": candidate.body,
|
|
||||||
"labels": label_ids,
|
|
||||||
"milestone": milestone_id,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
number = created.get("number") or created.get("index")
|
|
||||||
print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}")
|
|
||||||
state.fingerprints.add(candidate.fingerprint)
|
|
||||||
state.titles.add(normalize_title(candidate.title))
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
|
||||||
class RepoState:
|
|
||||||
target: Any
|
|
||||||
client: GiteaClient
|
|
||||||
label_ids: dict[str, int]
|
|
||||||
milestone_ids: dict[str, int]
|
|
||||||
fingerprints: set[str]
|
|
||||||
titles: set[str]
|
|
||||||
|
|
||||||
|
|
||||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
|
||||||
repos: dict[str, RepoInfo] = {}
|
|
||||||
for gitdir in sorted(git_root.glob("*/.git")):
|
|
||||||
root = gitdir.parent
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
remote = result.stdout.strip()
|
|
||||||
if "git.add-ideas.de" not in remote:
|
|
||||||
continue
|
|
||||||
target = infer_target(root)
|
|
||||||
repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo)
|
|
||||||
return repos
|
|
||||||
|
|
||||||
|
|
||||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]:
|
|
||||||
sources: list[SourceFile] = []
|
|
||||||
for repo in repos.values():
|
|
||||||
for path in repo.root.rglob("*"):
|
|
||||||
if not path.is_file() or not BACKLOG_NAME_RE.search(path.name):
|
|
||||||
continue
|
|
||||||
path_text = path.as_posix()
|
|
||||||
if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS):
|
|
||||||
continue
|
|
||||||
if is_excluded_repo_file(path):
|
|
||||||
continue
|
|
||||||
if is_text_backlog(path):
|
|
||||||
sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo"))
|
|
||||||
|
|
||||||
product_map = {
|
|
||||||
"co2api": "emission-api-lib",
|
|
||||||
"govoplan": "govoplan-core",
|
|
||||||
"meubility": "meubility-workbench",
|
|
||||||
"mousehold": "mousehold",
|
|
||||||
"prvnncr": "prvnncr-server",
|
|
||||||
}
|
|
||||||
if products_root.exists():
|
|
||||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
|
||||||
repo_key = product_map.get(product_dir.name)
|
|
||||||
if not repo_key or repo_key not in repos:
|
|
||||||
continue
|
|
||||||
for path in product_dir.rglob("*"):
|
|
||||||
if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path):
|
|
||||||
sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product"))
|
|
||||||
|
|
||||||
unique: dict[tuple[str, pathlib.Path], SourceFile] = {}
|
|
||||||
for source in sources:
|
|
||||||
unique[(source.repo_key, source.path.resolve())] = source
|
|
||||||
return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path)))
|
|
||||||
|
|
||||||
|
|
||||||
def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
|
||||||
text = path.as_posix()
|
|
||||||
name = path.name.lower()
|
|
||||||
if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"):
|
|
||||||
return True
|
|
||||||
if "testing_plan" in name or "test_plan" in name:
|
|
||||||
return True
|
|
||||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
|
||||||
return True
|
|
||||||
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
|
||||||
return True
|
|
||||||
if text.endswith("/govoplan-web/TODO.md"):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_text_backlog(path: pathlib.Path) -> bool:
|
|
||||||
if path.suffix.lower() not in {".md", ".txt", ".csv"}:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
chunk = path.read_bytes()[:4096]
|
|
||||||
except OSError:
|
|
||||||
return False
|
|
||||||
if b"\x00" in chunk:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]:
|
|
||||||
for source in sources:
|
|
||||||
suffix = source.path.suffix.lower()
|
|
||||||
if suffix == ".csv":
|
|
||||||
yield from parse_csv_source(source)
|
|
||||||
else:
|
|
||||||
yield from parse_text_source(source)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_csv_source(source: SourceFile) -> Iterable[Candidate]:
|
|
||||||
with source.path.open("r", encoding="utf-8-sig", newline="") as handle:
|
|
||||||
reader = csv.DictReader(handle)
|
|
||||||
for index, row in enumerate(reader, start=2):
|
|
||||||
story = (row.get("story") or row.get("title") or row.get("name") or "").strip()
|
|
||||||
if not story:
|
|
||||||
continue
|
|
||||||
identifier = (row.get("id") or "").strip()
|
|
||||||
priority = normalize_priority(row.get("priority") or "")
|
|
||||||
milestone = (row.get("epic") or "Backlog").strip()
|
|
||||||
title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story)
|
|
||||||
fingerprint = make_fingerprint(source.path, index, source.repo_key, story)
|
|
||||||
body = "\n".join(
|
|
||||||
[
|
|
||||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
"Imported from a CSV backlog file.",
|
|
||||||
"",
|
|
||||||
f"- Source: `{source.path}`",
|
|
||||||
f"- Line: `{index}`",
|
|
||||||
f"- Source kind: `{source.source_kind}`",
|
|
||||||
f"- Milestone: `{milestone}`",
|
|
||||||
"",
|
|
||||||
"CSV row:",
|
|
||||||
"",
|
|
||||||
"```text",
|
|
||||||
", ".join(f"{key}={value}" for key, value in row.items()),
|
|
||||||
"```",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key=source.repo_key,
|
|
||||||
title=title,
|
|
||||||
body=body,
|
|
||||||
labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"),
|
|
||||||
priority=priority,
|
|
||||||
milestone=milestone,
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(source.path),
|
|
||||||
line=index,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_text_source(source: SourceFile) -> Iterable[Candidate]:
|
|
||||||
try:
|
|
||||||
lines = source.path.read_text(encoding="utf-8").splitlines()
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
lines = source.path.read_text(encoding="latin-1").splitlines()
|
|
||||||
|
|
||||||
headings: list[tuple[int, str]] = []
|
|
||||||
in_tasks = False
|
|
||||||
for index, line in enumerate(lines):
|
|
||||||
line_number = index + 1
|
|
||||||
heading = parse_heading(line)
|
|
||||||
if heading:
|
|
||||||
level, title = heading
|
|
||||||
headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level]
|
|
||||||
headings.append((level, title))
|
|
||||||
in_tasks = False
|
|
||||||
continue
|
|
||||||
|
|
||||||
if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE):
|
|
||||||
in_tasks = True
|
|
||||||
continue
|
|
||||||
|
|
||||||
item = parse_open_item(line)
|
|
||||||
if item is None and is_action_context(source.path, headings, in_tasks):
|
|
||||||
item = parse_plain_action_item(line)
|
|
||||||
if item is None:
|
|
||||||
continue
|
|
||||||
if not item or should_skip_item(item, headings):
|
|
||||||
continue
|
|
||||||
|
|
||||||
section = section_title(headings)
|
|
||||||
priority = priority_from_context(section, item)
|
|
||||||
milestone = milestone_from_context(source.path, section)
|
|
||||||
prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]"
|
|
||||||
title = format_title(prefix, item)
|
|
||||||
fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item)
|
|
||||||
body = "\n".join(
|
|
||||||
[
|
|
||||||
f"<!-- codex-generic-backlog-fingerprint:{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
"Imported from a backlog-like text file.",
|
|
||||||
"",
|
|
||||||
f"- Source: `{source.path}`",
|
|
||||||
f"- Line: `{line_number}`",
|
|
||||||
f"- Source kind: `{source.source_kind}`",
|
|
||||||
f"- Section: `{section or 'none'}`",
|
|
||||||
"",
|
|
||||||
"Imported item:",
|
|
||||||
"",
|
|
||||||
"```text",
|
|
||||||
item,
|
|
||||||
"```",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
yield Candidate(
|
|
||||||
repo_key=source.repo_key,
|
|
||||||
title=title,
|
|
||||||
body=body,
|
|
||||||
labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"),
|
|
||||||
priority=priority,
|
|
||||||
milestone=milestone,
|
|
||||||
fingerprint=fingerprint,
|
|
||||||
source=str(source.path),
|
|
||||||
line=line_number,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_heading(line: str) -> tuple[int, str] | None:
|
|
||||||
markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
|
|
||||||
if markdown:
|
|
||||||
return len(markdown.group(1)), clean_text(markdown.group(2))
|
|
||||||
underlined = re.match(r"^(.+?)\s*$", line)
|
|
||||||
if underlined and line.strip() and len(line.strip()) < 120:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_open_item(line: str) -> str | None:
|
|
||||||
patterns = [
|
|
||||||
r"^(?P<indent>\s*)[-*]\s+\[\s\]\s+(?P<text>.+?)\s*$",
|
|
||||||
r"^(?P<indent>\s*)[-*]\s+☐\s+(?P<text>.+?)\s*$",
|
|
||||||
r"^(?P<indent>\s*)\d+[.)]\s+☐\s+(?P<text>.+?)\s*$",
|
|
||||||
r"^(?P<indent>\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P<text>.+?)\s*$",
|
|
||||||
]
|
|
||||||
for pattern in patterns:
|
|
||||||
match = re.match(pattern, line)
|
|
||||||
if match and leading_width(match.group("indent")) == 0:
|
|
||||||
return clean_text(match.group("text"))
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_plain_action_item(line: str) -> str | None:
|
|
||||||
match = re.match(r"^(?P<indent>\s*)[-*]\s+(?P<text>.+?)\s*$", line)
|
|
||||||
if match and leading_width(match.group("indent")) == 0:
|
|
||||||
text = clean_text(match.group("text"))
|
|
||||||
if "✅" in text or text.startswith(("✅", "[x]", "[X]")):
|
|
||||||
return None
|
|
||||||
if ACTION_RE.match(text) or is_short_noun_task(text):
|
|
||||||
return text
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool:
|
|
||||||
if in_tasks:
|
|
||||||
return True
|
|
||||||
context = section_title(headings)
|
|
||||||
if not context:
|
|
||||||
return False
|
|
||||||
if SKIP_SECTION_RE.search(context):
|
|
||||||
return False
|
|
||||||
if ACTIVE_SECTION_RE.search(context):
|
|
||||||
return True
|
|
||||||
return path.name.lower() in {"todo.txt", "todo.md"}
|
|
||||||
|
|
||||||
|
|
||||||
def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool:
|
|
||||||
context = section_title(headings)
|
|
||||||
if "✅" in item or item.startswith(("✅", "[x]", "[X]")):
|
|
||||||
return True
|
|
||||||
if item.endswith((",", ";")):
|
|
||||||
return True
|
|
||||||
if SKIP_SECTION_RE.search(context):
|
|
||||||
return True
|
|
||||||
if len(item) < 4:
|
|
||||||
return True
|
|
||||||
if item.endswith(":") and len(item.split()) <= 6:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_short_noun_task(text: str) -> bool:
|
|
||||||
return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text)
|
|
||||||
|
|
||||||
|
|
||||||
def clean_text(text: str) -> str:
|
|
||||||
text = text.strip()
|
|
||||||
text = re.sub(r"^☐\s*", "", text)
|
|
||||||
text = re.sub(r"^✅\s*", "", text)
|
|
||||||
text = re.sub(r"\s+", " ", text)
|
|
||||||
return text.strip(" -")
|
|
||||||
|
|
||||||
|
|
||||||
def leading_width(indent: str) -> int:
|
|
||||||
return len(indent.replace("\t", " "))
|
|
||||||
|
|
||||||
|
|
||||||
def section_title(headings: list[tuple[int, str]]) -> str:
|
|
||||||
return " > ".join(title for _, title in headings)
|
|
||||||
|
|
||||||
|
|
||||||
def priority_from_context(section: str, item: str) -> str:
|
|
||||||
text = f"{section} {item}".lower()
|
|
||||||
if "p0" in text or "critical path" in text:
|
|
||||||
return "priority/p0"
|
|
||||||
if "p1" in text or "immediate" in text or "next sprint" in text:
|
|
||||||
return "priority/p1"
|
|
||||||
if "p2" in text:
|
|
||||||
return "priority/p2"
|
|
||||||
if "p3" in text or "future" in text or "later" in text:
|
|
||||||
return "priority/p3"
|
|
||||||
return "priority/p2"
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_priority(value: str) -> str:
|
|
||||||
value = value.strip().lower()
|
|
||||||
if value == "p0":
|
|
||||||
return "priority/p0"
|
|
||||||
if value == "p1":
|
|
||||||
return "priority/p1"
|
|
||||||
if value == "p2":
|
|
||||||
return "priority/p2"
|
|
||||||
if value == "p3":
|
|
||||||
return "priority/p3"
|
|
||||||
return "priority/p2"
|
|
||||||
|
|
||||||
|
|
||||||
def milestone_from_context(path: pathlib.Path, section: str) -> str:
|
|
||||||
if section:
|
|
||||||
parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)]
|
|
||||||
if parts:
|
|
||||||
return parts[-1][:120]
|
|
||||||
return path.stem.replace("_", " ").replace("-", " ").title()
|
|
||||||
|
|
||||||
|
|
||||||
def format_title(prefix: str, item: str) -> str:
|
|
||||||
title = f"{prefix} {clean_text(item).rstrip('.')}"
|
|
||||||
if len(title) <= 180:
|
|
||||||
return title
|
|
||||||
return f"{title[:177].rstrip()}..."
|
|
||||||
|
|
||||||
|
|
||||||
def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str:
|
|
||||||
stable = "\n".join([str(path.resolve()), str(line), repo_key, item])
|
|
||||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
|
||||||
|
|
||||||
|
|
||||||
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
|
|
||||||
seen: set[tuple[str, str]] = set()
|
|
||||||
unique: list[Candidate] = []
|
|
||||||
for candidate in candidates:
|
|
||||||
key = (candidate.repo_key, normalize_title(candidate.title))
|
|
||||||
if key in seen:
|
|
||||||
continue
|
|
||||||
seen.add(key)
|
|
||||||
unique.append(candidate)
|
|
||||||
return unique
|
|
||||||
|
|
||||||
|
|
||||||
def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
|
||||||
counts: dict[str, int] = {}
|
|
||||||
for candidate in candidates:
|
|
||||||
counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1
|
|
||||||
return dict(sorted(counts.items()))
|
|
||||||
|
|
||||||
|
|
||||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
|
||||||
target = infer_target(repo.root)
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
|
||||||
if apply:
|
|
||||||
for name, color, description in GENERIC_LABELS:
|
|
||||||
if name in label_ids:
|
|
||||||
continue
|
|
||||||
created = client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(target.owner, target.repo, "/labels"),
|
|
||||||
body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))},
|
|
||||||
)
|
|
||||||
label_ids[str(created["name"])] = int(created["id"])
|
|
||||||
print(f"created label {target.owner}/{target.repo}:{name}")
|
|
||||||
|
|
||||||
missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids]
|
|
||||||
if missing and apply:
|
|
||||||
raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.")
|
|
||||||
|
|
||||||
milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50)
|
|
||||||
milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones}
|
|
||||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50)
|
|
||||||
fingerprints: set[str] = set()
|
|
||||||
titles: set[str] = set()
|
|
||||||
for issue in issues:
|
|
||||||
body = str(issue.get("body") or "")
|
|
||||||
titles.add(normalize_title(str(issue.get("title") or "")))
|
|
||||||
fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body))
|
|
||||||
return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles)
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_milestone(state: RepoState, title: str) -> int:
|
|
||||||
if title in state.milestone_ids:
|
|
||||||
return state.milestone_ids[title]
|
|
||||||
created = state.client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(state.target.owner, state.target.repo, "/milestones"),
|
|
||||||
body={"title": title, "state": "open", "description": "Created from imported backlog-like files."},
|
|
||||||
)
|
|
||||||
state.milestone_ids[title] = int(created["id"])
|
|
||||||
print(f"created milestone {state.target.owner}/{state.target.repo}:{title}")
|
|
||||||
return state.milestone_ids[title]
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_title(title: str) -> str:
|
|
||||||
return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None:
|
|
||||||
by_repo: dict[str, list[Candidate]] = {}
|
|
||||||
for candidate in candidates:
|
|
||||||
by_repo.setdefault(candidate.repo_key, []).append(candidate)
|
|
||||||
for repo_key, items in sorted(by_repo.items()):
|
|
||||||
print(f"{repo_key}:")
|
|
||||||
for item in items[:limit_per_repo]:
|
|
||||||
print(f" {item.title} [{item.priority}; {item.milestone}]")
|
|
||||||
if len(items) > limit_per_repo:
|
|
||||||
print(f" ... {len(items) - limit_per_repo} more")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Install Gitea issue workflow files into one or more repositories."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from gitea_common import GiteaError, repo_root
|
|
||||||
|
|
||||||
|
|
||||||
SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
WORKFLOW_FILES = (
|
|
||||||
".gitea/PULL_REQUEST_TEMPLATE.md",
|
|
||||||
".gitea/ISSUE_TEMPLATE/bug_report.md",
|
|
||||||
".gitea/ISSUE_TEMPLATE/config.yaml",
|
|
||||||
".gitea/ISSUE_TEMPLATE/docs_workflow.md",
|
|
||||||
".gitea/ISSUE_TEMPLATE/feature_request.md",
|
|
||||||
".gitea/ISSUE_TEMPLATE/task.md",
|
|
||||||
".gitea/ISSUE_TEMPLATE/tech_debt.md",
|
|
||||||
)
|
|
||||||
MODULE_LABEL_BY_REPO = {
|
|
||||||
"govoplan-core": "module/core",
|
|
||||||
"govoplan-access": "module/access",
|
|
||||||
"govoplan-admin": "module/admin",
|
|
||||||
"govoplan-tenancy": "module/tenancy",
|
|
||||||
"govoplan-policy": "module/policy",
|
|
||||||
"govoplan-audit": "module/audit",
|
|
||||||
"govoplan-mail": "module/mail",
|
|
||||||
"govoplan-files": "module/files",
|
|
||||||
"govoplan-campaign": "module/campaign",
|
|
||||||
"govoplan-web": "module/web",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths")
|
|
||||||
parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target")
|
|
||||||
parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json")
|
|
||||||
parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])]
|
|
||||||
unique_roots = list(dict.fromkeys(target_roots))
|
|
||||||
if args.module_label and len(unique_roots) != 1:
|
|
||||||
raise GiteaError("--module-label can only be used with a single target")
|
|
||||||
|
|
||||||
rel_paths = list(WORKFLOW_FILES)
|
|
||||||
if args.include_labels_file:
|
|
||||||
rel_paths.append("docs/gitea-labels.json")
|
|
||||||
|
|
||||||
for target_root in unique_roots:
|
|
||||||
module_label = args.module_label or infer_module_label(target_root)
|
|
||||||
install_files(target_root, rel_paths, module_label, apply=args.apply)
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
print("Dry run only. Re-run with --apply to write files.")
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def infer_module_label(target_root: pathlib.Path) -> str:
|
|
||||||
repo_name = target_root.name
|
|
||||||
try:
|
|
||||||
repo_name = target_root.resolve().name
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
label = MODULE_LABEL_BY_REPO.get(repo_name)
|
|
||||||
if label:
|
|
||||||
return label
|
|
||||||
if repo_name.startswith("govoplan-"):
|
|
||||||
suffix = repo_name.removeprefix("govoplan-")
|
|
||||||
if suffix:
|
|
||||||
return f"module/{suffix}"
|
|
||||||
raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/<name>.")
|
|
||||||
|
|
||||||
|
|
||||||
def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None:
|
|
||||||
print(f"Target: {target_root} ({module_label})")
|
|
||||||
for rel_path in rel_paths:
|
|
||||||
source = SOURCE_ROOT / rel_path
|
|
||||||
target = target_root / rel_path
|
|
||||||
if not source.exists():
|
|
||||||
raise GiteaError(f"missing workflow source file: {source}")
|
|
||||||
|
|
||||||
content = source.read_text(encoding="utf-8")
|
|
||||||
content = transform_content(rel_path, content, module_label)
|
|
||||||
|
|
||||||
existing = target.read_text(encoding="utf-8") if target.exists() else None
|
|
||||||
if existing == content:
|
|
||||||
print(f"unchanged {rel_path}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
action = "create" if existing is None else "update"
|
|
||||||
print(f"{action if apply else 'would ' + action} {rel_path}")
|
|
||||||
if apply:
|
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
target.write_text(content, encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def transform_content(rel_path: str, content: str, module_label: str) -> str:
|
|
||||||
if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"):
|
|
||||||
return content.replace(" - module/core", f" - {module_label}")
|
|
||||||
return content
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,250 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Move issue labels from repository-local labels to organization labels."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from gitea_common import (
|
|
||||||
GiteaClient,
|
|
||||||
GiteaError,
|
|
||||||
infer_target,
|
|
||||||
load_dotenv,
|
|
||||||
org_path,
|
|
||||||
repo_path,
|
|
||||||
repo_root,
|
|
||||||
require_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
|
||||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
|
||||||
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
|
||||||
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
|
||||||
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
|
||||||
parser.add_argument(
|
|
||||||
"--issue-mode",
|
|
||||||
choices=("replace", "delete-add"),
|
|
||||||
default="replace",
|
|
||||||
help="replace labels in one request, or delete local ids before adding org ids",
|
|
||||||
)
|
|
||||||
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
|
||||||
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
root = repo_root(args.root)
|
|
||||||
load_dotenv(args.env_file or root / ".env")
|
|
||||||
target = infer_target(root, args.remote)
|
|
||||||
owner = args.org or target.owner
|
|
||||||
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
|
||||||
if client.token is None:
|
|
||||||
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
|
||||||
|
|
||||||
repo_filter = re.compile(args.repo_regex)
|
|
||||||
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
|
||||||
if not org_labels:
|
|
||||||
raise GiteaError(f"{owner} has no organization labels")
|
|
||||||
|
|
||||||
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
|
||||||
totals = Totals()
|
|
||||||
print(f"Organization: {owner}")
|
|
||||||
print(f"Repositories: {len(repos)}")
|
|
||||||
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
|
||||||
print(f"Delete repository labels: {args.delete_repo_labels}")
|
|
||||||
|
|
||||||
for index, repo in enumerate(repos, start=1):
|
|
||||||
repo_result = migrate_repository(
|
|
||||||
client,
|
|
||||||
owner=owner,
|
|
||||||
repo=repo,
|
|
||||||
org_labels=org_labels,
|
|
||||||
issue_mode=args.issue_mode,
|
|
||||||
apply=args.apply,
|
|
||||||
delete_repo_labels=args.delete_repo_labels,
|
|
||||||
)
|
|
||||||
totals.add(repo_result)
|
|
||||||
if repo_result.has_work:
|
|
||||||
print(
|
|
||||||
f"[{index}/{len(repos)}] {repo}: "
|
|
||||||
f"issue-labels={repo_result.issue_label_migrations}, "
|
|
||||||
f"delete-labels={repo_result.repo_label_deletions}, "
|
|
||||||
f"errors={len(repo_result.errors)}"
|
|
||||||
)
|
|
||||||
for error in repo_result.errors:
|
|
||||||
print(f" error: {error}")
|
|
||||||
else:
|
|
||||||
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
|
||||||
|
|
||||||
print("Summary:")
|
|
||||||
print(f" repositories processed: {totals.repositories}")
|
|
||||||
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
|
||||||
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
|
||||||
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
|
||||||
print(f" errors: {totals.errors}")
|
|
||||||
if totals.errors:
|
|
||||||
return 1
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
class RepoResult:
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.issue_label_migrations = 0
|
|
||||||
self.repo_label_deletions = 0
|
|
||||||
self.errors: list[str] = []
|
|
||||||
self.matched_local_labels = 0
|
|
||||||
|
|
||||||
@property
|
|
||||||
def has_work(self) -> bool:
|
|
||||||
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
|
||||||
|
|
||||||
|
|
||||||
class Totals:
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.repositories = 0
|
|
||||||
self.repositories_with_work = 0
|
|
||||||
self.issue_label_migrations = 0
|
|
||||||
self.repo_label_deletions = 0
|
|
||||||
self.errors = 0
|
|
||||||
|
|
||||||
def add(self, result: RepoResult) -> None:
|
|
||||||
self.repositories += 1
|
|
||||||
if result.has_work:
|
|
||||||
self.repositories_with_work += 1
|
|
||||||
self.issue_label_migrations += result.issue_label_migrations
|
|
||||||
self.repo_label_deletions += result.repo_label_deletions
|
|
||||||
self.errors += len(result.errors)
|
|
||||||
|
|
||||||
|
|
||||||
def migrate_repository(
|
|
||||||
client: GiteaClient,
|
|
||||||
*,
|
|
||||||
owner: str,
|
|
||||||
repo: str,
|
|
||||||
org_labels: dict[str, dict[str, Any]],
|
|
||||||
issue_mode: str,
|
|
||||||
apply: bool,
|
|
||||||
delete_repo_labels: bool,
|
|
||||||
) -> RepoResult:
|
|
||||||
result = RepoResult()
|
|
||||||
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
|
||||||
local_labels = {
|
|
||||||
str(label.get("name") or ""): label
|
|
||||||
for label in repo_labels
|
|
||||||
if str(label.get("name") or "") in org_labels
|
|
||||||
}
|
|
||||||
result.matched_local_labels = len(local_labels)
|
|
||||||
if not local_labels:
|
|
||||||
return result
|
|
||||||
|
|
||||||
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
|
||||||
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
|
||||||
|
|
||||||
for issue_type in ("issues", "pulls"):
|
|
||||||
issues = client.paginate(
|
|
||||||
repo_path(owner, repo, "/issues"),
|
|
||||||
query={"state": "all", "type": issue_type},
|
|
||||||
limit=100,
|
|
||||||
)
|
|
||||||
for issue in issues:
|
|
||||||
issue_number = int(issue.get("number") or issue.get("index"))
|
|
||||||
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
|
||||||
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
|
||||||
final_label_ids = list(issue_label_ids)
|
|
||||||
changed = False
|
|
||||||
migrated_count = 0
|
|
||||||
local_ids_to_remove: list[int] = []
|
|
||||||
org_ids_to_add: list[int] = []
|
|
||||||
for label in issue.get("labels") or []:
|
|
||||||
local_id = _label_id(label)
|
|
||||||
if local_id is None or local_id not in local_by_id:
|
|
||||||
continue
|
|
||||||
name = str(label.get("name") or "")
|
|
||||||
org_id = org_by_name.get(name)
|
|
||||||
if org_id is None:
|
|
||||||
continue
|
|
||||||
local_ids_to_remove.append(local_id)
|
|
||||||
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
|
||||||
org_ids_to_add.append(org_id)
|
|
||||||
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
|
||||||
if org_id not in final_label_ids:
|
|
||||||
final_label_ids.append(org_id)
|
|
||||||
changed = True
|
|
||||||
migrated_count += 1
|
|
||||||
if changed:
|
|
||||||
if apply:
|
|
||||||
if issue_mode == "delete-add":
|
|
||||||
for local_id in local_ids_to_remove:
|
|
||||||
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
|
||||||
if org_ids_to_add:
|
|
||||||
client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
|
||||||
body={"labels": org_ids_to_add},
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
client.request_json(
|
|
||||||
"PUT",
|
|
||||||
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
|
||||||
body={"labels": final_label_ids},
|
|
||||||
)
|
|
||||||
result.issue_label_migrations += migrated_count
|
|
||||||
|
|
||||||
if delete_repo_labels:
|
|
||||||
for name, label in sorted(local_labels.items()):
|
|
||||||
label_id = _label_id(label)
|
|
||||||
if label_id is None:
|
|
||||||
result.errors.append(f"{name} has no label id")
|
|
||||||
continue
|
|
||||||
if apply:
|
|
||||||
try:
|
|
||||||
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
|
||||||
except GiteaError as exc:
|
|
||||||
result.errors.append(f"delete repository label {name}: {exc}")
|
|
||||||
continue
|
|
||||||
result.repo_label_deletions += 1
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
|
||||||
if explicit:
|
|
||||||
return sorted(set(explicit))
|
|
||||||
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
|
||||||
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
|
||||||
return [name for name in names if repo_filter.search(name)]
|
|
||||||
|
|
||||||
|
|
||||||
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
||||||
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
|
||||||
|
|
||||||
|
|
||||||
def _label_id(label: dict[str, Any]) -> int | None:
|
|
||||||
value = label.get("id") or label.get("index")
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
return int(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
|
||||||
try:
|
|
||||||
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
|
||||||
except GiteaError as exc:
|
|
||||||
if "HTTP 404" in str(exc):
|
|
||||||
return
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Synchronize issue labels to a Gitea repository."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from gitea_common import (
|
|
||||||
GiteaClient,
|
|
||||||
GiteaError,
|
|
||||||
infer_target,
|
|
||||||
load_dotenv,
|
|
||||||
load_json,
|
|
||||||
org_path,
|
|
||||||
repo_path,
|
|
||||||
repo_root,
|
|
||||||
require_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
|
||||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
|
||||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
||||||
parser.add_argument(
|
|
||||||
"--scope",
|
|
||||||
choices=("repository", "organization"),
|
|
||||||
default="repository",
|
|
||||||
help="sync repository labels or organization labels",
|
|
||||||
)
|
|
||||||
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
|
||||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
root = repo_root(args.root)
|
|
||||||
load_dotenv(args.env_file or root / ".env")
|
|
||||||
target = infer_target(root, args.remote)
|
|
||||||
labels = _load_labels(args.labels_file)
|
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
||||||
org_name = args.org or target.owner
|
|
||||||
|
|
||||||
if args.scope == "organization":
|
|
||||||
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
|
||||||
else:
|
|
||||||
print(f"Target: {target.display}")
|
|
||||||
print(f"Labels file: {args.labels_file}")
|
|
||||||
|
|
||||||
if not args.apply and not token:
|
|
||||||
print("Dry run without GITEA_TOKEN: API comparison skipped.")
|
|
||||||
for label in labels:
|
|
||||||
print(f"would ensure {label['name']} ({label['color']})")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
if args.scope == "organization":
|
|
||||||
existing = _org_labels_by_name(client, org_name)
|
|
||||||
else:
|
|
||||||
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
|
||||||
creates: list[dict[str, Any]] = []
|
|
||||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
|
||||||
|
|
||||||
for label in labels:
|
|
||||||
current = existing.get(label["name"])
|
|
||||||
if current is None:
|
|
||||||
creates.append(label)
|
|
||||||
continue
|
|
||||||
|
|
||||||
patch = _diff_label(current, label)
|
|
||||||
if patch:
|
|
||||||
updates.append((current, patch))
|
|
||||||
|
|
||||||
if not creates and not updates:
|
|
||||||
print("No label changes needed.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
for label in creates:
|
|
||||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
|
||||||
if args.apply:
|
|
||||||
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
|
||||||
|
|
||||||
for current, patch in updates:
|
|
||||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
|
||||||
if args.apply:
|
|
||||||
label_id = current.get("id") or current.get("index")
|
|
||||||
if label_id is None:
|
|
||||||
raise GiteaError(f"{current['name']} has no label id in API response")
|
|
||||||
client.request_json(
|
|
||||||
"PATCH",
|
|
||||||
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
|
||||||
body=patch,
|
|
||||||
)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
|
||||||
payload = load_json(path)
|
|
||||||
if not isinstance(payload, list):
|
|
||||||
raise GiteaError(f"{path} must contain a JSON list")
|
|
||||||
|
|
||||||
labels: list[dict[str, Any]] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
for index, item in enumerate(payload, start=1):
|
|
||||||
if not isinstance(item, dict):
|
|
||||||
raise GiteaError(f"label #{index} must be an object")
|
|
||||||
label = {
|
|
||||||
"name": str(item.get("name", "")).strip(),
|
|
||||||
"color": str(item.get("color", "")).strip().lstrip("#").lower(),
|
|
||||||
"description": str(item.get("description", "")).strip(),
|
|
||||||
"exclusive": bool(item.get("exclusive", False)),
|
|
||||||
}
|
|
||||||
if not label["name"]:
|
|
||||||
raise GiteaError(f"label #{index} is missing name")
|
|
||||||
if label["name"] in seen:
|
|
||||||
raise GiteaError(f"duplicate label name: {label['name']}")
|
|
||||||
if not _is_hex_color(label["color"]):
|
|
||||||
raise GiteaError(f"{label['name']} has invalid color {label['color']!r}")
|
|
||||||
seen.add(label["name"])
|
|
||||||
labels.append(label)
|
|
||||||
return labels
|
|
||||||
|
|
||||||
|
|
||||||
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
|
||||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
|
||||||
return {str(label.get("name")): label for label in labels}
|
|
||||||
|
|
||||||
|
|
||||||
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
|
||||||
labels = client.paginate(org_path(owner, "/labels"))
|
|
||||||
return {str(label.get("name")): label for label in labels}
|
|
||||||
|
|
||||||
|
|
||||||
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
|
||||||
if scope == "organization":
|
|
||||||
return org_path(org, "/labels")
|
|
||||||
return repo_path(owner, repo, "/labels")
|
|
||||||
|
|
||||||
|
|
||||||
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
|
||||||
if scope == "organization":
|
|
||||||
return org_path(org, f"/labels/{label_id}")
|
|
||||||
return repo_path(owner, repo, f"/labels/{label_id}")
|
|
||||||
|
|
||||||
|
|
||||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
patch: dict[str, Any] = {}
|
|
||||||
if _normalize_color(current.get("color")) != desired["color"]:
|
|
||||||
patch["color"] = desired["color"]
|
|
||||||
if str(current.get("description") or "").strip() != desired["description"]:
|
|
||||||
patch["description"] = desired["description"]
|
|
||||||
if bool(current.get("exclusive", False)) != desired["exclusive"]:
|
|
||||||
patch["exclusive"] = desired["exclusive"]
|
|
||||||
return patch
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_color(value: Any) -> str:
|
|
||||||
return str(value or "").strip().lstrip("#").lower()
|
|
||||||
|
|
||||||
|
|
||||||
def _is_hex_color(value: str) -> bool:
|
|
||||||
if len(value) != 6:
|
|
||||||
return False
|
|
||||||
return all(char in "0123456789abcdef" for char in value)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,637 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Mirror project documentation files into Gitea repository wikis."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
import dataclasses
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import urllib.parse
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
|
||||||
|
|
||||||
|
|
||||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
|
||||||
PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products")
|
|
||||||
MANAGED_MARKER = "<!-- codex-wiki-sync:"
|
|
||||||
TEXT_SUFFIXES = {".md", ".txt", ".csv", ".toml", ".json", ".yaml", ".yml", ".rst"}
|
|
||||||
SENSITIVE_NAME_RE = re.compile(r"(password|passwd|secret|credential|api[_-]?keys?|token|private[_-]?key)", re.IGNORECASE)
|
|
||||||
EXCLUDED_PARTS = {
|
|
||||||
".git",
|
|
||||||
".gitea",
|
|
||||||
".venv",
|
|
||||||
"node_modules",
|
|
||||||
"__pycache__",
|
|
||||||
"dist",
|
|
||||||
"build",
|
|
||||||
".cache",
|
|
||||||
".module-test-build",
|
|
||||||
}
|
|
||||||
EXCLUDED_NAMES = {
|
|
||||||
"package-lock.json",
|
|
||||||
"pnpm-lock.yaml",
|
|
||||||
"yarn.lock",
|
|
||||||
"uv.lock",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class RepoInfo:
|
|
||||||
root: pathlib.Path
|
|
||||||
owner: str
|
|
||||||
repo: str
|
|
||||||
|
|
||||||
@property
|
|
||||||
def key(self) -> str:
|
|
||||||
return self.root.name
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class WikiSource:
|
|
||||||
repo_key: str
|
|
||||||
path: pathlib.Path
|
|
||||||
origin: str
|
|
||||||
page_title: str
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
|
||||||
parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT)
|
|
||||||
parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT)
|
|
||||||
parser.add_argument("--repo", action="append", help="limit to one or more local repository directory names")
|
|
||||||
parser.add_argument("--page", action="append", help="limit to one or more generated wiki page names")
|
|
||||||
parser.add_argument("--overwrite-unmanaged", action="store_true", help="update existing wiki pages not previously managed by this script")
|
|
||||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
|
||||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
|
||||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
|
||||||
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
|
||||||
parser.add_argument("--apply", action="store_true")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
load_dotenv(args.env_file)
|
|
||||||
repos = discover_hosted_repos(args.git_root)
|
|
||||||
if args.repo:
|
|
||||||
requested = set(args.repo)
|
|
||||||
repos = {key: repo for key, repo in repos.items() if key in requested}
|
|
||||||
missing = requested - set(repos)
|
|
||||||
if missing:
|
|
||||||
raise GiteaError(f"requested repos are not hosted on git.add-ideas.de locally: {', '.join(sorted(missing))}")
|
|
||||||
|
|
||||||
sources = discover_sources(repos, args.products_root)
|
|
||||||
if args.page:
|
|
||||||
requested_pages = set(args.page)
|
|
||||||
sources = [source for source in sources if source.page_title in requested_pages]
|
|
||||||
missing_pages = requested_pages - {source.page_title for source in sources}
|
|
||||||
if missing_pages:
|
|
||||||
raise GiteaError(f"requested pages were not discovered: {', '.join(sorted(missing_pages))}")
|
|
||||||
print(f"Hosted repositories: {len(repos)}")
|
|
||||||
print(f"Wiki source files: {len(sources)}")
|
|
||||||
for repo_key, count in grouped_counts(sources).items():
|
|
||||||
print(f" {repo_key}: {count}")
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
preview_sources(sources)
|
|
||||||
print("Dry run only. Re-run with --apply to write wiki pages.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
token = require_token() if args.transport == "api" else ""
|
|
||||||
failures = 0
|
|
||||||
for repo_key, repo_sources in group_sources(sources).items():
|
|
||||||
repo = repos[repo_key]
|
|
||||||
try:
|
|
||||||
if args.transport == "git":
|
|
||||||
sync_repo_wiki_git(
|
|
||||||
repo,
|
|
||||||
repo_sources,
|
|
||||||
cache_dir=args.wiki_cache_dir,
|
|
||||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
|
||||||
commit_message=args.commit_message,
|
|
||||||
write_index=not bool(args.page),
|
|
||||||
prune_managed=args.prune_managed and not bool(args.page),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
sync_repo_wiki(
|
|
||||||
repo,
|
|
||||||
repo_sources,
|
|
||||||
token,
|
|
||||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
|
||||||
write_index=not bool(args.page),
|
|
||||||
prune_managed=args.prune_managed and not bool(args.page),
|
|
||||||
)
|
|
||||||
except GiteaError as exc:
|
|
||||||
failures += 1
|
|
||||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
|
||||||
return 1 if failures else 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]:
|
|
||||||
repos: dict[str, RepoInfo] = {}
|
|
||||||
for gitdir in sorted(git_root.glob("*/.git")):
|
|
||||||
root = gitdir.parent
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "remote", "get-url", "origin"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if "git.add-ideas.de" not in result.stdout:
|
|
||||||
continue
|
|
||||||
target = infer_target(root)
|
|
||||||
repos[root.name] = RepoInfo(root=root, owner=target.owner, repo=target.repo)
|
|
||||||
return repos
|
|
||||||
|
|
||||||
|
|
||||||
def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[WikiSource]:
|
|
||||||
sources: list[WikiSource] = []
|
|
||||||
for repo in repos.values():
|
|
||||||
for path in repo.root.rglob("*"):
|
|
||||||
if is_repo_doc(repo.root, path):
|
|
||||||
rel = path.relative_to(repo.root)
|
|
||||||
sources.append(
|
|
||||||
WikiSource(
|
|
||||||
repo_key=repo.key,
|
|
||||||
path=path,
|
|
||||||
origin="repository",
|
|
||||||
page_title=title_for_path("Repo", rel),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
product_map = {
|
|
||||||
"co2api": "emission-api-lib",
|
|
||||||
"govoplan": "govoplan-core",
|
|
||||||
"meubility": "meubility-workbench",
|
|
||||||
"mousehold": "mousehold",
|
|
||||||
"prvnncr": "prvnncr-server",
|
|
||||||
}
|
|
||||||
if products_root.exists():
|
|
||||||
for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()):
|
|
||||||
repo_key = product_map.get(product_dir.name)
|
|
||||||
if not repo_key or repo_key not in repos:
|
|
||||||
continue
|
|
||||||
for path in product_dir.rglob("*"):
|
|
||||||
if is_product_doc(path):
|
|
||||||
rel = path.relative_to(product_dir)
|
|
||||||
sources.append(
|
|
||||||
WikiSource(
|
|
||||||
repo_key=repo_key,
|
|
||||||
path=path,
|
|
||||||
origin=f"product:{product_dir.name}",
|
|
||||||
page_title=title_for_path(f"Product {product_dir.name}", rel),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
unique: dict[tuple[str, str], WikiSource] = {}
|
|
||||||
for source in sources:
|
|
||||||
unique[(source.repo_key, source.page_title)] = source
|
|
||||||
return sorted(unique.values(), key=lambda item: (item.repo_key, item.page_title))
|
|
||||||
|
|
||||||
|
|
||||||
def is_repo_doc(repo_root_path: pathlib.Path, path: pathlib.Path) -> bool:
|
|
||||||
if not path.is_file() or not is_text_file(path):
|
|
||||||
return False
|
|
||||||
parts = set(path.relative_to(repo_root_path).parts)
|
|
||||||
if parts & EXCLUDED_PARTS:
|
|
||||||
return False
|
|
||||||
if path.name in EXCLUDED_NAMES:
|
|
||||||
return False
|
|
||||||
if SENSITIVE_NAME_RE.search(path.name):
|
|
||||||
return False
|
|
||||||
rel = path.relative_to(repo_root_path)
|
|
||||||
if len(rel.parts) == 1 and path.name.lower().startswith(("readme", "license", "changelog", "contributing", "security")):
|
|
||||||
return True
|
|
||||||
if "generated" in rel.parts:
|
|
||||||
return False
|
|
||||||
if rel.parts[0] in {"docs", "doc", "codex"} and path.suffix.lower() in {".md", ".txt", ".csv"}:
|
|
||||||
return True
|
|
||||||
if re.search(r"(backlog|todo|roadmap|plan|architecture|workflow|manifest)", path.name, re.IGNORECASE):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_product_doc(path: pathlib.Path) -> bool:
|
|
||||||
if not path.is_file() or not is_text_file(path):
|
|
||||||
return False
|
|
||||||
if path.name in EXCLUDED_NAMES:
|
|
||||||
return False
|
|
||||||
if SENSITIVE_NAME_RE.search(path.name):
|
|
||||||
return False
|
|
||||||
if any(part in {"python_backup", "bruno", "fluege"} for part in path.parts):
|
|
||||||
return False
|
|
||||||
if re.search(
|
|
||||||
r"(readme|todo|roadmap|plan|concept|continuation|copyright|notes|whitepaper|pitch|request_response)",
|
|
||||||
path.name,
|
|
||||||
re.IGNORECASE,
|
|
||||||
) and path.suffix.lower() in {".md", ".txt"}:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_text_file(path: pathlib.Path) -> bool:
|
|
||||||
if path.suffix.lower() not in TEXT_SUFFIXES:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
chunk = path.read_bytes()[:4096]
|
|
||||||
except OSError:
|
|
||||||
return False
|
|
||||||
return b"\x00" not in chunk
|
|
||||||
|
|
||||||
|
|
||||||
def title_for_path(prefix: str, rel: pathlib.Path) -> str:
|
|
||||||
stem = rel.with_suffix("").as_posix()
|
|
||||||
stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-")
|
|
||||||
return f"{prefix}-{stem}"[:180]
|
|
||||||
|
|
||||||
|
|
||||||
def group_sources(sources: list[WikiSource]) -> dict[str, list[WikiSource]]:
|
|
||||||
grouped: dict[str, list[WikiSource]] = {}
|
|
||||||
for source in sources:
|
|
||||||
grouped.setdefault(source.repo_key, []).append(source)
|
|
||||||
return dict(sorted(grouped.items()))
|
|
||||||
|
|
||||||
|
|
||||||
def grouped_counts(sources: list[WikiSource]) -> dict[str, int]:
|
|
||||||
return {repo_key: len(items) for repo_key, items in group_sources(sources).items()}
|
|
||||||
|
|
||||||
|
|
||||||
def preview_sources(sources: list[WikiSource]) -> None:
|
|
||||||
for repo_key, repo_sources in group_sources(sources).items():
|
|
||||||
print(f"{repo_key}:")
|
|
||||||
for source in repo_sources[:80]:
|
|
||||||
print(f" {source.page_title} <- {source.path}")
|
|
||||||
if len(repo_sources) > 80:
|
|
||||||
print(f" ... {len(repo_sources) - 80} more")
|
|
||||||
|
|
||||||
|
|
||||||
def sync_repo_wiki(
|
|
||||||
repo: RepoInfo,
|
|
||||||
sources: list[WikiSource],
|
|
||||||
token: str,
|
|
||||||
*,
|
|
||||||
overwrite_unmanaged: bool,
|
|
||||||
write_index: bool = True,
|
|
||||||
prune_managed: bool = False,
|
|
||||||
) -> None:
|
|
||||||
target = infer_target(repo.root)
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
|
||||||
existing_page_names = {
|
|
||||||
str(page.get("title")): str(page.get("sub_url") or page.get("title"))
|
|
||||||
for page in existing_pages
|
|
||||||
if page.get("title")
|
|
||||||
}
|
|
||||||
index_lines = [
|
|
||||||
f"# {target.repo} Project Wiki Index",
|
|
||||||
"",
|
|
||||||
f"{MANAGED_MARKER}index -->",
|
|
||||||
"",
|
|
||||||
"This page is generated from repository and product-directory project files.",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
for source in sources:
|
|
||||||
content = render_wiki_content(source)
|
|
||||||
changed = put_wiki_page(
|
|
||||||
client,
|
|
||||||
target.owner,
|
|
||||||
target.repo,
|
|
||||||
source.page_title,
|
|
||||||
content,
|
|
||||||
existing_page_names,
|
|
||||||
overwrite_unmanaged=overwrite_unmanaged,
|
|
||||||
)
|
|
||||||
print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}")
|
|
||||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
|
||||||
|
|
||||||
if write_index:
|
|
||||||
put_wiki_page(
|
|
||||||
client,
|
|
||||||
target.owner,
|
|
||||||
target.repo,
|
|
||||||
"Codex-Project-Index",
|
|
||||||
"\n".join(index_lines) + "\n",
|
|
||||||
existing_page_names,
|
|
||||||
overwrite_unmanaged=True,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
|
||||||
if prune_managed and write_index:
|
|
||||||
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
|
||||||
|
|
||||||
|
|
||||||
def sync_repo_wiki_git(
|
|
||||||
repo: RepoInfo,
|
|
||||||
sources: list[WikiSource],
|
|
||||||
*,
|
|
||||||
cache_dir: pathlib.Path,
|
|
||||||
overwrite_unmanaged: bool,
|
|
||||||
commit_message: str,
|
|
||||||
write_index: bool = True,
|
|
||||||
prune_managed: bool = False,
|
|
||||||
) -> None:
|
|
||||||
target = infer_target(repo.root)
|
|
||||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
|
||||||
index_lines = [
|
|
||||||
f"# {target.repo} Project Wiki Index",
|
|
||||||
"",
|
|
||||||
f"{MANAGED_MARKER}index -->",
|
|
||||||
"",
|
|
||||||
"This page is generated from repository and product-directory project files.",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
for source in sources:
|
|
||||||
content = render_wiki_content(source)
|
|
||||||
status = write_wiki_page_file(
|
|
||||||
wiki_root,
|
|
||||||
source.page_title,
|
|
||||||
content,
|
|
||||||
overwrite_unmanaged=overwrite_unmanaged,
|
|
||||||
)
|
|
||||||
print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}")
|
|
||||||
index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`")
|
|
||||||
|
|
||||||
if write_index:
|
|
||||||
write_wiki_page_file(
|
|
||||||
wiki_root,
|
|
||||||
"Codex-Project-Index",
|
|
||||||
"\n".join(index_lines) + "\n",
|
|
||||||
overwrite_unmanaged=True,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
|
||||||
if prune_managed and write_index:
|
|
||||||
prune_managed_wiki_pages_git(wiki_root, sources)
|
|
||||||
if not git_has_changes(wiki_root):
|
|
||||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
|
||||||
return
|
|
||||||
run_git(wiki_root, "add", "-A")
|
|
||||||
if not git_has_staged_changes(wiki_root):
|
|
||||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
|
||||||
return
|
|
||||||
ensure_git_identity(wiki_root)
|
|
||||||
run_git(wiki_root, "commit", "-m", commit_message)
|
|
||||||
run_git(wiki_root, "push")
|
|
||||||
print(f"pushed {target.owner}/{target.repo} wiki repository")
|
|
||||||
|
|
||||||
|
|
||||||
def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path:
|
|
||||||
remote = wiki_remote_for_repo(repo.root)
|
|
||||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki"
|
|
||||||
if (checkout / ".git").exists():
|
|
||||||
run_git(checkout, "remote", "set-url", "origin", remote)
|
|
||||||
run_git(checkout, "fetch", "origin")
|
|
||||||
branch = current_branch(checkout)
|
|
||||||
if branch:
|
|
||||||
run_git(checkout, "reset", "--hard", f"origin/{branch}")
|
|
||||||
else:
|
|
||||||
run_git(checkout, "reset", "--hard")
|
|
||||||
run_git(checkout, "clean", "-fd")
|
|
||||||
return checkout
|
|
||||||
if checkout.exists():
|
|
||||||
shutil.rmtree(checkout)
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "clone", remote, str(checkout)],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode == 0:
|
|
||||||
return checkout
|
|
||||||
checkout.mkdir(parents=True, exist_ok=True)
|
|
||||||
run_git(checkout, "init")
|
|
||||||
run_git(checkout, "remote", "add", "origin", remote)
|
|
||||||
run_git(checkout, "checkout", "-b", "master")
|
|
||||||
return checkout
|
|
||||||
|
|
||||||
|
|
||||||
def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(repo_root_path), "remote", "get-url", "origin"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0 or not result.stdout.strip():
|
|
||||||
raise GiteaError(f"Could not read origin remote for {repo_root_path}")
|
|
||||||
remote = result.stdout.strip()
|
|
||||||
if remote.endswith(".git"):
|
|
||||||
return f"{remote[:-4]}.wiki.git"
|
|
||||||
return f"{remote}.wiki.git"
|
|
||||||
|
|
||||||
|
|
||||||
def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str:
|
|
||||||
path = wiki_root / f"{wiki_file_stem(title)}.md"
|
|
||||||
if path.exists():
|
|
||||||
current = read_text(path)
|
|
||||||
if current == content:
|
|
||||||
return "unchanged"
|
|
||||||
if MANAGED_MARKER not in current and not overwrite_unmanaged:
|
|
||||||
return "skipped unmanaged"
|
|
||||||
path.write_text(content, encoding="utf-8")
|
|
||||||
return "updated"
|
|
||||||
|
|
||||||
|
|
||||||
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
|
||||||
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
|
||||||
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
|
||||||
for path in sorted(wiki_root.glob("*.md")):
|
|
||||||
if path.name in desired:
|
|
||||||
continue
|
|
||||||
current = read_text(path)
|
|
||||||
if MANAGED_MARKER not in current:
|
|
||||||
continue
|
|
||||||
path.unlink()
|
|
||||||
print(f"deleted stale managed wiki page {path.name}")
|
|
||||||
|
|
||||||
|
|
||||||
def prune_managed_wiki_pages_api(
|
|
||||||
client: GiteaClient,
|
|
||||||
owner: str,
|
|
||||||
repo: str,
|
|
||||||
existing_page_names: dict[str, str],
|
|
||||||
sources: list[WikiSource],
|
|
||||||
) -> None:
|
|
||||||
desired = {source.page_title for source in sources}
|
|
||||||
desired.add("Codex-Project-Index")
|
|
||||||
for title, page_name in sorted(existing_page_names.items()):
|
|
||||||
if title in desired:
|
|
||||||
continue
|
|
||||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
|
||||||
if MANAGED_MARKER not in decode_wiki_content(current):
|
|
||||||
continue
|
|
||||||
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
|
||||||
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
|
||||||
|
|
||||||
|
|
||||||
def wiki_file_stem(title: str) -> str:
|
|
||||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
|
||||||
|
|
||||||
|
|
||||||
def git_has_changes(root: pathlib.Path) -> bool:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "status", "--porcelain"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}")
|
|
||||||
return bool(result.stdout.strip())
|
|
||||||
|
|
||||||
|
|
||||||
def git_has_staged_changes(root: pathlib.Path) -> bool:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "diff", "--cached", "--quiet"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode == 0:
|
|
||||||
return False
|
|
||||||
if result.returncode == 1:
|
|
||||||
return True
|
|
||||||
raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}")
|
|
||||||
|
|
||||||
|
|
||||||
def current_branch(root: pathlib.Path) -> str:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
branch = result.stdout.strip()
|
|
||||||
if result.returncode != 0 or branch == "HEAD":
|
|
||||||
return ""
|
|
||||||
return branch
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_git_identity(root: pathlib.Path) -> None:
|
|
||||||
if not git_config_value(root, "user.email"):
|
|
||||||
run_git(root, "config", "user.email", "codex@govoplan.local")
|
|
||||||
if not git_config_value(root, "user.name"):
|
|
||||||
run_git(root, "config", "user.name", "Codex")
|
|
||||||
|
|
||||||
|
|
||||||
def git_config_value(root: pathlib.Path, key: str) -> str:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "config", "--get", key],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
return result.stdout.strip() if result.returncode == 0 else ""
|
|
||||||
|
|
||||||
|
|
||||||
def run_git(root: pathlib.Path, *args: str) -> None:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), *args],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
command = "git -C " + str(root) + " " + " ".join(args)
|
|
||||||
raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}")
|
|
||||||
|
|
||||||
|
|
||||||
def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]:
|
|
||||||
try:
|
|
||||||
return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50)
|
|
||||||
except GiteaError as exc:
|
|
||||||
if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc):
|
|
||||||
return []
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def render_wiki_content(source: WikiSource) -> str:
|
|
||||||
raw = read_text(source.path)
|
|
||||||
fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
|
||||||
header = [
|
|
||||||
f"{MANAGED_MARKER}{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
f"> Mirrored from `{source.path}`.",
|
|
||||||
f"> Origin: `{source.origin}`.",
|
|
||||||
"> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.",
|
|
||||||
"",
|
|
||||||
"---",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
if source.path.suffix.lower() == ".csv":
|
|
||||||
return "\n".join(header + ["```csv", raw.rstrip(), "```", ""])
|
|
||||||
return "\n".join(header) + raw.rstrip() + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def put_wiki_page(
|
|
||||||
client: GiteaClient,
|
|
||||||
owner: str,
|
|
||||||
repo: str,
|
|
||||||
title: str,
|
|
||||||
content: str,
|
|
||||||
existing_page_names: dict[str, str],
|
|
||||||
*,
|
|
||||||
overwrite_unmanaged: bool,
|
|
||||||
) -> bool:
|
|
||||||
body = {
|
|
||||||
"title": title,
|
|
||||||
"content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"),
|
|
||||||
"message": f"Sync {title} from project files",
|
|
||||||
}
|
|
||||||
if title in existing_page_names:
|
|
||||||
page_name = existing_page_names[title]
|
|
||||||
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
|
||||||
current_content = decode_wiki_content(current)
|
|
||||||
if current_content == content:
|
|
||||||
return False
|
|
||||||
if MANAGED_MARKER not in current_content and not overwrite_unmanaged:
|
|
||||||
print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}")
|
|
||||||
return False
|
|
||||||
client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body)
|
|
||||||
return True
|
|
||||||
client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body)
|
|
||||||
existing_page_names[title] = title
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def quote_wiki_page_name(value: str) -> str:
|
|
||||||
return urllib.parse.quote(value, safe="+")
|
|
||||||
|
|
||||||
|
|
||||||
def decode_wiki_content(page: dict[str, Any]) -> str:
|
|
||||||
encoded = str(page.get("content_base64") or "")
|
|
||||||
if not encoded:
|
|
||||||
return ""
|
|
||||||
return base64.b64decode(encoded).decode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def read_text(path: pathlib.Path) -> str:
|
|
||||||
try:
|
|
||||||
return path.read_text(encoding="utf-8")
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
return path.read_text(encoding="latin-1")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,377 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Preview or import inline TODO-style markers as Gitea issues."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
|
||||||
|
|
||||||
|
|
||||||
MARKER_RE = re.compile(
|
|
||||||
r"(?P<prefix>#|//|/\*|\*|--|<!--)?\s*"
|
|
||||||
r"\b(?P<marker>TODO|FIXME|XXX|HACK)\b"
|
|
||||||
r"\s*(?:\((?P<context>[^)]{1,120})\))?"
|
|
||||||
r"\s*(?P<colon>:)?\s*(?P<text>.*)",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
DEFAULT_EXCLUDES = (
|
|
||||||
"!**/.git/**",
|
|
||||||
"!**/.venv/**",
|
|
||||||
"!**/node_modules/**",
|
|
||||||
"!**/__pycache__/**",
|
|
||||||
"!**/.module-test-build/**",
|
|
||||||
"!**/dist/**",
|
|
||||||
"!**/build/**",
|
|
||||||
"!**/.cache/**",
|
|
||||||
"!.gitea/**",
|
|
||||||
"!docs/GITEA_ISSUES.md",
|
|
||||||
"!scripts/gitea-todo-import.py",
|
|
||||||
"!scripts/gitea-sync-labels.py",
|
|
||||||
"!scripts/gitea-codex-note.py",
|
|
||||||
"!scripts/gitea_common.py",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path")
|
|
||||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
|
||||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
|
||||||
parser.add_argument("--include-linked", action="store_true", help="include markers that already reference an issue")
|
|
||||||
parser.add_argument(
|
|
||||||
"--module-label",
|
|
||||||
help="project/module label to apply; defaults to a known GovOPlaN mapping or module/core",
|
|
||||||
)
|
|
||||||
parser.add_argument("--no-area-labels", action="store_true", help="do not infer area/* labels from paths")
|
|
||||||
parser.add_argument("--extra-label", action="append", default=[], help="additional label name to apply")
|
|
||||||
parser.add_argument("--limit", type=int, default=0, help="maximum number of markers to process")
|
|
||||||
parser.add_argument("--json", action="store_true", help="print issue previews as JSON")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
root = repo_root(args.root)
|
|
||||||
load_dotenv(args.env_file or root / ".env")
|
|
||||||
target = infer_target(root, args.remote)
|
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
|
||||||
|
|
||||||
markers = scan_markers(root, include_linked=args.include_linked)
|
|
||||||
if args.limit > 0:
|
|
||||||
markers = markers[: args.limit]
|
|
||||||
|
|
||||||
module_label = args.module_label or infer_module_label(root)
|
|
||||||
previews = [
|
|
||||||
build_issue_preview(
|
|
||||||
target.owner,
|
|
||||||
target.repo,
|
|
||||||
marker,
|
|
||||||
module_label=module_label,
|
|
||||||
infer_areas=not args.no_area_labels,
|
|
||||||
extra_labels=args.extra_label,
|
|
||||||
)
|
|
||||||
for marker in markers
|
|
||||||
]
|
|
||||||
|
|
||||||
if args.json:
|
|
||||||
print(json.dumps([preview.as_dict() for preview in previews], indent=2, sort_keys=True))
|
|
||||||
else:
|
|
||||||
print(f"Target: {target.display}")
|
|
||||||
print(f"Scanned root: {root}")
|
|
||||||
print(f"Found {len(previews)} importable marker(s).")
|
|
||||||
for preview in previews:
|
|
||||||
print(f"- {preview.title}")
|
|
||||||
print(f" {preview.location}")
|
|
||||||
print(f" labels: {', '.join(preview.labels)}")
|
|
||||||
|
|
||||||
if not args.apply:
|
|
||||||
if not previews:
|
|
||||||
return 0
|
|
||||||
print("Dry run only. Re-run with --apply and GITEA_TOKEN to create issues.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if not previews:
|
|
||||||
print("No issues to create.")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
client = GiteaClient(target, token)
|
|
||||||
label_ids = load_label_ids(client, target.owner, target.repo)
|
|
||||||
missing_labels = sorted({label for preview in previews for label in preview.labels if label not in label_ids})
|
|
||||||
if missing_labels:
|
|
||||||
joined = ", ".join(missing_labels)
|
|
||||||
raise GiteaError(f"missing labels: {joined}. Run scripts/gitea-sync-labels.py --apply first.")
|
|
||||||
|
|
||||||
existing_fingerprints = load_existing_fingerprints(client, target.owner, target.repo)
|
|
||||||
created = 0
|
|
||||||
skipped = 0
|
|
||||||
for preview in previews:
|
|
||||||
if preview.fingerprint in existing_fingerprints:
|
|
||||||
print(f"skip existing {preview.location}")
|
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
issue = client.request_json(
|
|
||||||
"POST",
|
|
||||||
repo_path(target.owner, target.repo, "/issues"),
|
|
||||||
body={
|
|
||||||
"title": preview.title,
|
|
||||||
"body": preview.body,
|
|
||||||
"labels": [label_ids[label] for label in preview.labels],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
print(f"created #{issue.get('number') or issue.get('index')}: {preview.title}")
|
|
||||||
created += 1
|
|
||||||
|
|
||||||
print(f"Created {created} issue(s), skipped {skipped} existing issue(s).")
|
|
||||||
return 0
|
|
||||||
except GiteaError as exc:
|
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
class Marker:
|
|
||||||
def __init__(self, path: pathlib.Path, line: int, column: int, marker: str, context: str, text: str, raw: str) -> None:
|
|
||||||
self.path = path
|
|
||||||
self.line = line
|
|
||||||
self.column = column
|
|
||||||
self.marker = marker.upper()
|
|
||||||
self.context = context
|
|
||||||
self.text = text
|
|
||||||
self.raw = raw.rstrip("\n")
|
|
||||||
|
|
||||||
@property
|
|
||||||
def location(self) -> str:
|
|
||||||
return f"{self.path}:{self.line}"
|
|
||||||
|
|
||||||
|
|
||||||
class IssuePreview:
|
|
||||||
def __init__(self, title: str, body: str, labels: list[str], fingerprint: str, location: str) -> None:
|
|
||||||
self.title = title
|
|
||||||
self.body = body
|
|
||||||
self.labels = labels
|
|
||||||
self.fingerprint = fingerprint
|
|
||||||
self.location = location
|
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"title": self.title,
|
|
||||||
"body": self.body,
|
|
||||||
"labels": self.labels,
|
|
||||||
"fingerprint": self.fingerprint,
|
|
||||||
"location": self.location,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def scan_markers(root: pathlib.Path, *, include_linked: bool) -> list[Marker]:
|
|
||||||
command = [
|
|
||||||
"rg",
|
|
||||||
"--json",
|
|
||||||
"--hidden",
|
|
||||||
"--line-number",
|
|
||||||
"--column",
|
|
||||||
"-e",
|
|
||||||
r"\b(TODO|FIXME|XXX|HACK)\b",
|
|
||||||
]
|
|
||||||
for pattern in DEFAULT_EXCLUDES:
|
|
||||||
command.extend(["--glob", pattern])
|
|
||||||
command.append(str(root))
|
|
||||||
|
|
||||||
result = subprocess.run(
|
|
||||||
command,
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode not in {0, 1}:
|
|
||||||
raise GiteaError(f"rg failed: {result.stderr.strip()}")
|
|
||||||
|
|
||||||
markers: list[Marker] = []
|
|
||||||
for line in result.stdout.splitlines():
|
|
||||||
event = json.loads(line)
|
|
||||||
if event.get("type") != "match":
|
|
||||||
continue
|
|
||||||
data = event["data"]
|
|
||||||
raw_line = data["lines"]["text"]
|
|
||||||
match = MARKER_RE.search(raw_line)
|
|
||||||
if not match:
|
|
||||||
continue
|
|
||||||
if not _looks_like_marker(match):
|
|
||||||
continue
|
|
||||||
context = (match.group("context") or "").strip()
|
|
||||||
if not include_linked and _already_linked(context, raw_line):
|
|
||||||
continue
|
|
||||||
markers.append(
|
|
||||||
Marker(
|
|
||||||
path=pathlib.Path(data["path"]["text"]).resolve().relative_to(root),
|
|
||||||
line=int(data["line_number"]),
|
|
||||||
column=int(data.get("submatches", [{}])[0].get("start", 0)) + 1,
|
|
||||||
marker=match.group("marker"),
|
|
||||||
context=context,
|
|
||||||
text=clean_marker_text(match.group("text") or raw_line),
|
|
||||||
raw=raw_line,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return markers
|
|
||||||
|
|
||||||
|
|
||||||
def build_issue_preview(
|
|
||||||
owner: str,
|
|
||||||
repo: str,
|
|
||||||
marker: Marker,
|
|
||||||
*,
|
|
||||||
module_label: str,
|
|
||||||
infer_areas: bool,
|
|
||||||
extra_labels: list[str],
|
|
||||||
) -> IssuePreview:
|
|
||||||
fingerprint = marker_fingerprint(owner, repo, marker)
|
|
||||||
summary = marker.text or f"review {marker.location}"
|
|
||||||
title = truncate_title(f"{marker.marker}: {summary} ({marker.location})")
|
|
||||||
labels = sorted(set(default_labels(marker, module_label=module_label, infer_areas=infer_areas) + list(extra_labels)))
|
|
||||||
body = "\n".join(
|
|
||||||
[
|
|
||||||
f"<!-- codex-todo-fingerprint:{fingerprint} -->",
|
|
||||||
"",
|
|
||||||
"Imported from an inline marker.",
|
|
||||||
"",
|
|
||||||
f"- Repository: `{owner}/{repo}`",
|
|
||||||
f"- Location: `{marker.location}`",
|
|
||||||
f"- Marker: `{marker.marker}`",
|
|
||||||
f"- Context: `{marker.context or 'none'}`",
|
|
||||||
"",
|
|
||||||
"Current line:",
|
|
||||||
"",
|
|
||||||
"```text",
|
|
||||||
marker.raw,
|
|
||||||
"```",
|
|
||||||
"",
|
|
||||||
"Migration rule:",
|
|
||||||
"",
|
|
||||||
"- Keep this Gitea issue as the canonical backlog item.",
|
|
||||||
"- When touching this code, remove the inline marker or replace it with a short issue reference.",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location)
|
|
||||||
|
|
||||||
|
|
||||||
def clean_marker_text(text: str) -> str:
|
|
||||||
cleaned = text.strip()
|
|
||||||
cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip()
|
|
||||||
cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip()
|
|
||||||
return cleaned
|
|
||||||
|
|
||||||
|
|
||||||
def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]:
|
|
||||||
labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)]
|
|
||||||
area = infer_area_label(marker.path) if infer_areas else ""
|
|
||||||
if area:
|
|
||||||
labels.append(area)
|
|
||||||
return labels
|
|
||||||
|
|
||||||
|
|
||||||
def marker_type_label(marker: str) -> str:
|
|
||||||
if marker == "FIXME":
|
|
||||||
return "type/bug"
|
|
||||||
if marker in {"HACK", "XXX"}:
|
|
||||||
return "type/debt"
|
|
||||||
return "type/task"
|
|
||||||
|
|
||||||
|
|
||||||
def infer_area_label(path: pathlib.Path) -> str:
|
|
||||||
text = path.as_posix()
|
|
||||||
if text.startswith("webui/"):
|
|
||||||
return "area/webui"
|
|
||||||
if text.startswith("alembic/"):
|
|
||||||
return "area/migrations"
|
|
||||||
if text.startswith("docs/"):
|
|
||||||
return "area/docs"
|
|
||||||
if text.startswith("scripts/"):
|
|
||||||
return "area/devex"
|
|
||||||
if "/rbac" in text or "permission" in text:
|
|
||||||
return "area/rbac"
|
|
||||||
if "/access" in text or "auth" in text or "session" in text:
|
|
||||||
return "area/auth"
|
|
||||||
if "tenant" in text:
|
|
||||||
return "area/tenancy"
|
|
||||||
if "governance" in text or "privacy" in text or "audit" in text or "retention" in text:
|
|
||||||
return "area/governance"
|
|
||||||
if "module" in text:
|
|
||||||
return "area/module-system"
|
|
||||||
if "migration" in text:
|
|
||||||
return "area/migrations"
|
|
||||||
if "router" in text or "api" in text:
|
|
||||||
return "area/api"
|
|
||||||
if "db" in text or "model" in text or "persistence" in text:
|
|
||||||
return "area/db"
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def infer_module_label(root: pathlib.Path) -> str:
|
|
||||||
known = {
|
|
||||||
"govoplan-core": "module/core",
|
|
||||||
"govoplan-access": "module/access",
|
|
||||||
"govoplan-admin": "module/admin",
|
|
||||||
"govoplan-tenancy": "module/tenancy",
|
|
||||||
"govoplan-policy": "module/policy",
|
|
||||||
"govoplan-audit": "module/audit",
|
|
||||||
"govoplan-mail": "module/mail",
|
|
||||||
"govoplan-files": "module/files",
|
|
||||||
"govoplan-campaign": "module/campaign",
|
|
||||||
"govoplan-web": "module/web",
|
|
||||||
}
|
|
||||||
if root.name in known:
|
|
||||||
return known[root.name]
|
|
||||||
if root.name.startswith("govoplan-"):
|
|
||||||
suffix = root.name.removeprefix("govoplan-")
|
|
||||||
if suffix:
|
|
||||||
return f"module/{suffix}"
|
|
||||||
return "module/core"
|
|
||||||
|
|
||||||
|
|
||||||
def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str:
|
|
||||||
stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text])
|
|
||||||
return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24]
|
|
||||||
|
|
||||||
|
|
||||||
def truncate_title(title: str, limit: int = 180) -> str:
|
|
||||||
if len(title) <= limit:
|
|
||||||
return title
|
|
||||||
return f"{title[: limit - 1].rstrip()}..."
|
|
||||||
|
|
||||||
|
|
||||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
|
||||||
return label_ids_by_name(client, owner, repo)
|
|
||||||
|
|
||||||
|
|
||||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
|
||||||
issues = client.paginate(
|
|
||||||
repo_path(owner, repo, "/issues"),
|
|
||||||
query={"state": "all", "labels": "source/todo-scan"},
|
|
||||||
)
|
|
||||||
fingerprints: set[str] = set()
|
|
||||||
for issue in issues:
|
|
||||||
body = str(issue.get("body") or "")
|
|
||||||
for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body):
|
|
||||||
fingerprints.add(match.group(1))
|
|
||||||
return fingerprints
|
|
||||||
|
|
||||||
|
|
||||||
def _already_linked(context: str, raw_line: str) -> bool:
|
|
||||||
linked_text = f"{context} {raw_line}".lower()
|
|
||||||
return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _looks_like_marker(match: re.Match[str]) -> bool:
|
|
||||||
text = (match.group("text") or "").strip()
|
|
||||||
return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,290 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Shared helpers for Gitea maintenance scripts."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import dataclasses
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import urllib.error
|
|
||||||
import urllib.parse
|
|
||||||
import urllib.request
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
class GiteaError(RuntimeError):
|
|
||||||
"""Raised when a Gitea API request fails."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class RepoTarget:
|
|
||||||
base_url: str
|
|
||||||
owner: str
|
|
||||||
repo: str
|
|
||||||
|
|
||||||
@property
|
|
||||||
def display(self) -> str:
|
|
||||||
return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def api_base(self) -> str:
|
|
||||||
return f"{self.base_url.rstrip('/')}/api/v1"
|
|
||||||
|
|
||||||
|
|
||||||
def repo_root(start: pathlib.Path) -> pathlib.Path:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(start), "rev-parse", "--show-toplevel"],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
return start.resolve()
|
|
||||||
return pathlib.Path(result.stdout.strip()).resolve()
|
|
||||||
|
|
||||||
|
|
||||||
def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget:
|
|
||||||
env_url = os.environ.get("GITEA_URL")
|
|
||||||
env_owner = os.environ.get("GITEA_OWNER")
|
|
||||||
env_repo = os.environ.get("GITEA_REPO")
|
|
||||||
|
|
||||||
remote_url = _git_remote(root, remote_name)
|
|
||||||
inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url)
|
|
||||||
|
|
||||||
base_url = (env_url or inferred_url).rstrip("/")
|
|
||||||
owner = env_owner or inferred_owner
|
|
||||||
repo = env_repo or inferred_repo
|
|
||||||
|
|
||||||
missing = [
|
|
||||||
name
|
|
||||||
for name, value in (
|
|
||||||
("GITEA_URL", base_url),
|
|
||||||
("GITEA_OWNER", owner),
|
|
||||||
("GITEA_REPO", repo),
|
|
||||||
)
|
|
||||||
if not value
|
|
||||||
]
|
|
||||||
if missing:
|
|
||||||
joined = ", ".join(missing)
|
|
||||||
raise GiteaError(
|
|
||||||
f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}."
|
|
||||||
)
|
|
||||||
return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo))
|
|
||||||
|
|
||||||
|
|
||||||
def quote_path(value: str) -> str:
|
|
||||||
return urllib.parse.quote(value, safe="")
|
|
||||||
|
|
||||||
|
|
||||||
class GiteaClient:
|
|
||||||
def __init__(self, target: RepoTarget, token: str | None) -> None:
|
|
||||||
self.target = target
|
|
||||||
self.token = token
|
|
||||||
|
|
||||||
def request_json(
|
|
||||||
self,
|
|
||||||
method: str,
|
|
||||||
path: str,
|
|
||||||
*,
|
|
||||||
body: dict[str, Any] | None = None,
|
|
||||||
query: dict[str, Any] | None = None,
|
|
||||||
) -> Any:
|
|
||||||
url = f"{self.target.api_base}{path}"
|
|
||||||
if query:
|
|
||||||
url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}"
|
|
||||||
|
|
||||||
data = None
|
|
||||||
headers = {
|
|
||||||
"Accept": "application/json",
|
|
||||||
"User-Agent": "codex-gitea-maintenance",
|
|
||||||
}
|
|
||||||
if self.token:
|
|
||||||
headers["Authorization"] = f"token {self.token}"
|
|
||||||
if body is not None:
|
|
||||||
data = json.dumps(body).encode("utf-8")
|
|
||||||
headers["Content-Type"] = "application/json"
|
|
||||||
|
|
||||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(request, timeout=30) as response:
|
|
||||||
payload = response.read().decode("utf-8")
|
|
||||||
if not payload:
|
|
||||||
return None
|
|
||||||
return json.loads(payload)
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
message = exc.read().decode("utf-8", errors="replace")
|
|
||||||
raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc
|
|
||||||
except urllib.error.URLError as exc:
|
|
||||||
raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc
|
|
||||||
|
|
||||||
def paginate(
|
|
||||||
self,
|
|
||||||
path: str,
|
|
||||||
*,
|
|
||||||
query: dict[str, Any] | None = None,
|
|
||||||
limit: int = 100,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
items: list[dict[str, Any]] = []
|
|
||||||
page = 1
|
|
||||||
effective_limit = min(limit, 50)
|
|
||||||
seen_pages: set[tuple[Any, ...]] = set()
|
|
||||||
while True:
|
|
||||||
page_query = dict(query or {})
|
|
||||||
page_query.update({"page": page, "limit": effective_limit})
|
|
||||||
payload = self.request_json("GET", path, query=page_query)
|
|
||||||
if not isinstance(payload, list):
|
|
||||||
raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}")
|
|
||||||
if not payload:
|
|
||||||
return items
|
|
||||||
signature = tuple(
|
|
||||||
item.get("id") or item.get("number") or item.get("index") or item.get("name")
|
|
||||||
for item in payload
|
|
||||||
if isinstance(item, dict)
|
|
||||||
)
|
|
||||||
if signature in seen_pages:
|
|
||||||
return items
|
|
||||||
seen_pages.add(signature)
|
|
||||||
items.extend(payload)
|
|
||||||
if len(payload) < effective_limit:
|
|
||||||
return items
|
|
||||||
page += 1
|
|
||||||
|
|
||||||
|
|
||||||
def load_json(path: pathlib.Path) -> Any:
|
|
||||||
with path.open("r", encoding="utf-8") as handle:
|
|
||||||
return json.load(handle)
|
|
||||||
|
|
||||||
|
|
||||||
def load_dotenv(path: pathlib.Path | None) -> list[str]:
|
|
||||||
if path is None or not path.exists():
|
|
||||||
return []
|
|
||||||
|
|
||||||
loaded: list[str] = []
|
|
||||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
||||||
stripped = line.strip()
|
|
||||||
if not stripped or stripped.startswith("#"):
|
|
||||||
continue
|
|
||||||
if stripped.startswith("export "):
|
|
||||||
stripped = stripped[len("export ") :].strip()
|
|
||||||
if "=" not in stripped:
|
|
||||||
raise GiteaError(f"{path}:{line_number}: expected KEY=value")
|
|
||||||
key, value = stripped.split("=", 1)
|
|
||||||
key = key.strip()
|
|
||||||
value = _strip_env_value(value.strip())
|
|
||||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
|
||||||
raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}")
|
|
||||||
if key not in os.environ:
|
|
||||||
os.environ[key] = value
|
|
||||||
loaded.append(key)
|
|
||||||
return loaded
|
|
||||||
|
|
||||||
|
|
||||||
def require_token() -> str:
|
|
||||||
token = os.environ.get("GITEA_TOKEN")
|
|
||||||
if not token:
|
|
||||||
raise GiteaError(
|
|
||||||
"GITEA_TOKEN is required when using --apply. "
|
|
||||||
"Set it in the environment, the target .env, or --env-file."
|
|
||||||
)
|
|
||||||
return token
|
|
||||||
|
|
||||||
|
|
||||||
def repo_path(owner: str, repo: str, suffix: str) -> str:
|
|
||||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
|
||||||
|
|
||||||
|
|
||||||
def org_path(owner: str, suffix: str) -> str:
|
|
||||||
return f"/orgs/{quote_path(owner)}{suffix}"
|
|
||||||
|
|
||||||
|
|
||||||
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
|
||||||
"""Return org labels plus optional repository labels, keyed by label name.
|
|
||||||
|
|
||||||
Gitea stores organization labels separately from repository labels. Issue
|
|
||||||
APIs can use label ids from both scopes on supported instances, so issue
|
|
||||||
creation helpers should resolve both. Repository labels intentionally win
|
|
||||||
when a repo carries a local label with the same name.
|
|
||||||
"""
|
|
||||||
|
|
||||||
labels: dict[str, dict[str, Any]] = {}
|
|
||||||
try:
|
|
||||||
for label in client.paginate(org_path(owner, "/labels")):
|
|
||||||
name = str(label.get("name") or "")
|
|
||||||
if name:
|
|
||||||
labels[name] = label
|
|
||||||
except GiteaError:
|
|
||||||
# User-owned repositories or older instances may not expose org labels.
|
|
||||||
pass
|
|
||||||
|
|
||||||
if repo:
|
|
||||||
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
|
||||||
name = str(label.get("name") or "")
|
|
||||||
if name:
|
|
||||||
labels[name] = label
|
|
||||||
return labels
|
|
||||||
|
|
||||||
|
|
||||||
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
|
||||||
ids: dict[str, int] = {}
|
|
||||||
for name, label in labels_by_name(client, owner, repo).items():
|
|
||||||
label_id = label.get("id") or label.get("index")
|
|
||||||
if label_id is None:
|
|
||||||
continue
|
|
||||||
ids[name] = int(label_id)
|
|
||||||
return ids
|
|
||||||
|
|
||||||
|
|
||||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
|
||||||
result = subprocess.run(
|
|
||||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
|
||||||
check=False,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
return ""
|
|
||||||
return result.stdout.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_remote(remote_url: str) -> tuple[str, str, str]:
|
|
||||||
if not remote_url:
|
|
||||||
return "", "", ""
|
|
||||||
|
|
||||||
parsed = urllib.parse.urlparse(remote_url)
|
|
||||||
if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc:
|
|
||||||
path_parts = [part for part in parsed.path.strip("/").split("/") if part]
|
|
||||||
owner, repo = _owner_repo_from_parts(path_parts)
|
|
||||||
if parsed.scheme in {"http", "https"}:
|
|
||||||
prefix = "/".join(path_parts[:-2])
|
|
||||||
base_path = f"/{prefix}" if prefix else ""
|
|
||||||
return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo
|
|
||||||
return f"https://{parsed.hostname or parsed.netloc}", owner, repo
|
|
||||||
|
|
||||||
scp_like = re.match(r"^(?:[^@]+@)?(?P<host>[^:]+):(?P<path>.+)$", remote_url)
|
|
||||||
if scp_like:
|
|
||||||
path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part]
|
|
||||||
owner, repo = _owner_repo_from_parts(path_parts)
|
|
||||||
return f"https://{scp_like.group('host')}", owner, repo
|
|
||||||
|
|
||||||
return "", "", ""
|
|
||||||
|
|
||||||
|
|
||||||
def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]:
|
|
||||||
if len(path_parts) < 2:
|
|
||||||
return "", ""
|
|
||||||
return path_parts[-2], _strip_git_suffix(path_parts[-1])
|
|
||||||
|
|
||||||
|
|
||||||
def _strip_git_suffix(value: str) -> str:
|
|
||||||
return value[:-4] if value.endswith(".git") else value
|
|
||||||
|
|
||||||
|
|
||||||
def _strip_env_value(value: str) -> str:
|
|
||||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
|
||||||
return value[1:-1]
|
|
||||||
return value
|
|
||||||
@@ -1,171 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
|
||||||
WEBUI_ROOT="$ROOT/webui"
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
|
||||||
NPM="${NPM:-$NODE_BIN/npm}"
|
|
||||||
|
|
||||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
|
||||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
|
||||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
|
||||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
|
||||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
|
||||||
FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
|
||||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
|
||||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
|
||||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
|
||||||
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
|
||||||
|
|
||||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
|
||||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
|
||||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
|
||||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
|
||||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
|
||||||
|
|
||||||
backend_pid=""
|
|
||||||
frontend_pid=""
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
printf 'launch-dev: %s\n' "$*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$DEV_DATABASE_BACKEND" in
|
|
||||||
postgres)
|
|
||||||
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
|
||||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
|
||||||
;;
|
|
||||||
sqlite)
|
|
||||||
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
|
||||||
|
|
||||||
port_is_free() {
|
|
||||||
"$PYTHON" - "$1" "$2" <<'PY'
|
|
||||||
import socket
|
|
||||||
import sys
|
|
||||||
|
|
||||||
host = sys.argv[1]
|
|
||||||
port = int(sys.argv[2])
|
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
||||||
try:
|
|
||||||
sock.bind((host, port))
|
|
||||||
except OSError:
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_url() {
|
|
||||||
"$PYTHON" - "$1" <<'PY'
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
url = sys.argv[1]
|
|
||||||
deadline = time.monotonic() + 60
|
|
||||||
last_error = None
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(url, timeout=2) as response:
|
|
||||||
if 200 <= response.status < 500:
|
|
||||||
raise SystemExit(0)
|
|
||||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
|
||||||
last_error = exc
|
|
||||||
time.sleep(1)
|
|
||||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
|
||||||
kill "$frontend_pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
|
||||||
kill "$backend_pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
trap cleanup EXIT INT TERM
|
|
||||||
|
|
||||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
|
||||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
|
||||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
|
||||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
|
||||||
|
|
||||||
mkdir -p "$LOG_DIR"
|
|
||||||
: > "$BACKEND_LOG"
|
|
||||||
: > "$FRONTEND_LOG"
|
|
||||||
|
|
||||||
if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then
|
|
||||||
rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp"
|
|
||||||
fi
|
|
||||||
|
|
||||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
|
||||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
|
||||||
|
|
||||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
|
||||||
(
|
|
||||||
cd "$ROOT"
|
|
||||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
|
||||||
) >"$BACKEND_LOG" 2>&1 &
|
|
||||||
backend_pid="$!"
|
|
||||||
|
|
||||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
|
||||||
wait_for_url "$BACKEND_URL/health" || {
|
|
||||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
|
||||||
fail "backend did not become healthy"
|
|
||||||
}
|
|
||||||
|
|
||||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
|
||||||
(
|
|
||||||
cd "$WEBUI_ROOT"
|
|
||||||
frontend_args=(run dev)
|
|
||||||
if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then
|
|
||||||
frontend_args+=(-- --force)
|
|
||||||
fi
|
|
||||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
|
||||||
CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \
|
|
||||||
CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \
|
|
||||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
|
||||||
"$NPM" "${frontend_args[@]}"
|
|
||||||
) >"$FRONTEND_LOG" 2>&1 &
|
|
||||||
frontend_pid="$!"
|
|
||||||
|
|
||||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
|
||||||
wait_for_url "$FRONTEND_URL" || {
|
|
||||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
|
||||||
fail "frontend did not become reachable"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
|
||||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
cat <<EOF
|
|
||||||
|
|
||||||
GovOPlaN is running.
|
|
||||||
Web UI: $FRONTEND_URL
|
|
||||||
API: $BACKEND_URL/api/v1
|
|
||||||
Health: $BACKEND_URL/health
|
|
||||||
DB: $DATABASE_URL
|
|
||||||
|
|
||||||
Logs:
|
|
||||||
Backend: $BACKEND_LOG
|
|
||||||
WebUI: $FRONTEND_LOG
|
|
||||||
|
|
||||||
Frontend reload:
|
|
||||||
Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE
|
|
||||||
Vite --force: $FRONTEND_FORCE_RELOAD
|
|
||||||
Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms)
|
|
||||||
|
|
||||||
Press Ctrl+C to stop both processes.
|
|
||||||
EOF
|
|
||||||
|
|
||||||
wait
|
|
||||||
@@ -1,241 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
|
||||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
|
||||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
|
||||||
if [ ! -f "$PROFILE_ENV" ]; then
|
|
||||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
|
||||||
fi
|
|
||||||
|
|
||||||
WEBUI_ROOT="$ROOT/webui"
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
|
||||||
NPM="${NPM:-$NODE_BIN/npm}"
|
|
||||||
DOCKER="${DOCKER:-docker}"
|
|
||||||
|
|
||||||
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
|
||||||
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
|
||||||
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
|
||||||
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
|
||||||
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
|
||||||
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
|
||||||
|
|
||||||
LOG_DIR="$ROOT/runtime/production-like/logs"
|
|
||||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
|
||||||
WORKER_LOG="$LOG_DIR/worker.log"
|
|
||||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
|
||||||
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
|
||||||
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
|
||||||
|
|
||||||
backend_pid=""
|
|
||||||
worker_pid=""
|
|
||||||
frontend_pid=""
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
|
||||||
|
|
||||||
set -a
|
|
||||||
# shellcheck disable=SC1090
|
|
||||||
. "$PROFILE_ENV"
|
|
||||||
set +a
|
|
||||||
|
|
||||||
export APP_ENV="${APP_ENV:-staging}"
|
|
||||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
|
||||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
|
||||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
|
||||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
|
||||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
|
||||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
|
||||||
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
|
||||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
|
||||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
|
||||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
|
||||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
|
||||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
|
||||||
|
|
||||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
|
||||||
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
|
||||||
from cryptography.fernet import Fernet
|
|
||||||
print(Fernet.generate_key().decode())
|
|
||||||
PY
|
|
||||||
)"
|
|
||||||
export MASTER_KEY_B64
|
|
||||||
fi
|
|
||||||
|
|
||||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
|
||||||
|
|
||||||
compose() {
|
|
||||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
port_is_free() {
|
|
||||||
"$PYTHON" - "$1" "$2" <<'PY'
|
|
||||||
import socket
|
|
||||||
import sys
|
|
||||||
|
|
||||||
host = sys.argv[1]
|
|
||||||
port = int(sys.argv[2])
|
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
||||||
try:
|
|
||||||
sock.bind((host, port))
|
|
||||||
except OSError:
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_tcp() {
|
|
||||||
"$PYTHON" - "$1" "$2" <<'PY'
|
|
||||||
import socket
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
|
|
||||||
host = sys.argv[1]
|
|
||||||
port = int(sys.argv[2])
|
|
||||||
deadline = time.monotonic() + 90
|
|
||||||
last_error = None
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
try:
|
|
||||||
with socket.create_connection((host, port), timeout=2):
|
|
||||||
raise SystemExit(0)
|
|
||||||
except OSError as exc:
|
|
||||||
last_error = exc
|
|
||||||
time.sleep(1)
|
|
||||||
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_url() {
|
|
||||||
"$PYTHON" - "$1" <<'PY'
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
url = sys.argv[1]
|
|
||||||
deadline = time.monotonic() + 90
|
|
||||||
last_error = None
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(url, timeout=2) as response:
|
|
||||||
if 200 <= response.status < 500:
|
|
||||||
raise SystemExit(0)
|
|
||||||
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
|
||||||
last_error = exc
|
|
||||||
time.sleep(1)
|
|
||||||
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
|
||||||
raise SystemExit(1)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
|
||||||
kill "$frontend_pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
|
||||||
kill "$backend_pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
|
||||||
kill "$worker_pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
|
||||||
compose down >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
trap cleanup EXIT INT TERM
|
|
||||||
|
|
||||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
|
||||||
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
|
||||||
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
|
||||||
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
|
||||||
|
|
||||||
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
|
||||||
: > "$BACKEND_LOG"
|
|
||||||
: > "$WORKER_LOG"
|
|
||||||
: > "$FRONTEND_LOG"
|
|
||||||
|
|
||||||
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
|
||||||
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
|
||||||
|
|
||||||
printf 'Starting production-like dependencies through Docker Compose\n'
|
|
||||||
compose up -d
|
|
||||||
|
|
||||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
|
||||||
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
|
||||||
|
|
||||||
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
|
||||||
(
|
|
||||||
cd "$ROOT"
|
|
||||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
|
||||||
)
|
|
||||||
|
|
||||||
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
|
||||||
(
|
|
||||||
cd "$ROOT"
|
|
||||||
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
|
||||||
) >"$BACKEND_LOG" 2>&1 &
|
|
||||||
backend_pid="$!"
|
|
||||||
|
|
||||||
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
|
||||||
wait_for_url "$BACKEND_URL/health" || {
|
|
||||||
tail -n 80 "$BACKEND_LOG" >&2 || true
|
|
||||||
fail "backend did not become healthy"
|
|
||||||
}
|
|
||||||
|
|
||||||
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
|
||||||
(
|
|
||||||
cd "$ROOT"
|
|
||||||
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
|
||||||
--queues "$CELERY_QUEUES" \
|
|
||||||
--hostname "govoplan-production-like@%h" \
|
|
||||||
--loglevel INFO
|
|
||||||
) >"$WORKER_LOG" 2>&1 &
|
|
||||||
worker_pid="$!"
|
|
||||||
|
|
||||||
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
|
||||||
(
|
|
||||||
cd "$WEBUI_ROOT"
|
|
||||||
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
|
||||||
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
|
||||||
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
|
||||||
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
|
||||||
"$NPM" run dev -- --force
|
|
||||||
) >"$FRONTEND_LOG" 2>&1 &
|
|
||||||
frontend_pid="$!"
|
|
||||||
|
|
||||||
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
|
||||||
wait_for_url "$FRONTEND_URL" || {
|
|
||||||
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
|
||||||
fail "frontend did not become reachable"
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
|
||||||
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
cat <<EOF
|
|
||||||
|
|
||||||
GovOPlaN production-like development profile is running.
|
|
||||||
Web UI: $FRONTEND_URL
|
|
||||||
API: $BACKEND_URL/api/v1
|
|
||||||
Health: $BACKEND_URL/health
|
|
||||||
Ops: $BACKEND_URL/api/v1/ops/status
|
|
||||||
DB: $DATABASE_URL
|
|
||||||
Redis: $REDIS_URL
|
|
||||||
Files: $FILE_STORAGE_LOCAL_ROOT
|
|
||||||
|
|
||||||
Logs:
|
|
||||||
Backend: $BACKEND_LOG
|
|
||||||
Worker: $WORKER_LOG
|
|
||||||
WebUI: $FRONTEND_LOG
|
|
||||||
|
|
||||||
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
|
||||||
Press Ctrl+C to stop API, worker, and WebUI.
|
|
||||||
EOF
|
|
||||||
|
|
||||||
wait
|
|
||||||
@@ -1,512 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import shlex
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from collections.abc import Callable, Iterable
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from typing import Any
|
|
||||||
from urllib.error import URLError
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
SRC = ROOT / "src"
|
|
||||||
if str(SRC) not in sys.path:
|
|
||||||
sys.path.insert(0, str(SRC))
|
|
||||||
|
|
||||||
os.environ.setdefault("APP_ENV", "test")
|
|
||||||
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
|
||||||
os.environ.setdefault("CELERY_ENABLED", "false")
|
|
||||||
|
|
||||||
from govoplan_core.admin.models import SystemSettings
|
|
||||||
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
|
||||||
from govoplan_core.core.module_installer import (
|
|
||||||
cancel_module_installer_request,
|
|
||||||
claim_next_module_installer_request,
|
|
||||||
module_installer_daemon_status,
|
|
||||||
module_installer_lock_status,
|
|
||||||
queue_module_installer_request,
|
|
||||||
read_module_installer_run,
|
|
||||||
retry_module_installer_request,
|
|
||||||
rollback_module_install_run,
|
|
||||||
run_module_install_plan,
|
|
||||||
supervise_module_install_plan,
|
|
||||||
update_module_installer_daemon_status,
|
|
||||||
update_module_installer_request,
|
|
||||||
)
|
|
||||||
from govoplan_core.core.module_management import (
|
|
||||||
ModuleInstallPlan,
|
|
||||||
ModuleInstallPlanItem,
|
|
||||||
saved_desired_enabled_modules,
|
|
||||||
saved_module_install_plan,
|
|
||||||
save_desired_enabled_modules,
|
|
||||||
save_module_install_plan,
|
|
||||||
)
|
|
||||||
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
|
||||||
from govoplan_core.db.base import Base
|
|
||||||
from govoplan_core.db.session import configure_database, get_database
|
|
||||||
from govoplan_core.server.registry import available_module_manifests
|
|
||||||
|
|
||||||
|
|
||||||
Scenario = Callable[[Path], dict[str, object]]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
|
||||||
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
|
||||||
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
|
||||||
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
|
||||||
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
|
||||||
runtime_root.mkdir(parents=True, exist_ok=True)
|
|
||||||
selected = args.scenario or sorted(SCENARIOS)
|
|
||||||
results: list[dict[str, object]] = []
|
|
||||||
ok = True
|
|
||||||
try:
|
|
||||||
for name in selected:
|
|
||||||
scenario_root = runtime_root / name
|
|
||||||
scenario_root.mkdir(parents=True, exist_ok=True)
|
|
||||||
try:
|
|
||||||
result = SCENARIOS[name](scenario_root)
|
|
||||||
except Exception as exc:
|
|
||||||
ok = False
|
|
||||||
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
|
||||||
else:
|
|
||||||
ok = ok and bool(result.get("ok"))
|
|
||||||
results.append(result)
|
|
||||||
finally:
|
|
||||||
if not args.keep_runtime and args.runtime_root is None:
|
|
||||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
|
||||||
|
|
||||||
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
|
||||||
if args.format == "json":
|
|
||||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
||||||
else:
|
|
||||||
for result in results:
|
|
||||||
status = "ok" if result.get("ok") else "FAILED"
|
|
||||||
print(f"{status}: {result['scenario']}")
|
|
||||||
if result.get("run_id"):
|
|
||||||
print(f" run: {result['run_id']}")
|
|
||||||
if result.get("record_path"):
|
|
||||||
print(f" record: {result['record_path']}")
|
|
||||||
if result.get("error"):
|
|
||||||
print(f" error: {result['error']}")
|
|
||||||
print(f"runtime: {runtime_root}")
|
|
||||||
return 0 if ok else 1
|
|
||||||
|
|
||||||
|
|
||||||
def package_failure(root: Path) -> dict[str, object]:
|
|
||||||
database_url, database = _sqlite_database(root)
|
|
||||||
plan = _saved_install_plan(database, "package-failure-example")
|
|
||||||
with database.session() as session, _patch_subprocess(_package_failure_run):
|
|
||||||
result = supervise_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available_module_manifests(),
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url=database_url,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
)
|
|
||||||
restored_plan = saved_module_install_plan(session)
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
return _scenario_result(
|
|
||||||
"package-failure",
|
|
||||||
root,
|
|
||||||
result,
|
|
||||||
expected_status="rolled-back",
|
|
||||||
checks={
|
|
||||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
|
||||||
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def migration_failure(root: Path) -> dict[str, object]:
|
|
||||||
database_url, database = _sqlite_database(root)
|
|
||||||
plan = _saved_install_plan(database, "migration-failure-example")
|
|
||||||
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
|
||||||
result = supervise_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available_module_manifests(),
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url=database_url,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
migrate_database=True,
|
|
||||||
)
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
return _scenario_result(
|
|
||||||
"migration-failure",
|
|
||||||
root,
|
|
||||||
result,
|
|
||||||
expected_status="rolled-back",
|
|
||||||
checks={
|
|
||||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
|
||||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def restart_failure(root: Path) -> dict[str, object]:
|
|
||||||
database_url, database = _sqlite_database(root)
|
|
||||||
plan = _saved_install_plan(database, "restart-failure-example")
|
|
||||||
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
|
||||||
result = supervise_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available_module_manifests(),
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url=database_url,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
restart_command="restart-fails",
|
|
||||||
)
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
return _scenario_result(
|
|
||||||
"restart-failure",
|
|
||||||
root,
|
|
||||||
result,
|
|
||||||
expected_status="rolled-back",
|
|
||||||
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def health_timeout(root: Path) -> dict[str, object]:
|
|
||||||
database_url, database = _sqlite_database(root)
|
|
||||||
plan = _saved_install_plan(database, "health-timeout-example")
|
|
||||||
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
|
||||||
result = supervise_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available_module_manifests(),
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url=database_url,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
restart_command="restart-ok",
|
|
||||||
health_url="http://127.0.0.1:9/health",
|
|
||||||
health_timeout_seconds=0.2,
|
|
||||||
health_interval_seconds=0.05,
|
|
||||||
)
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
return _scenario_result(
|
|
||||||
"health-timeout",
|
|
||||||
root,
|
|
||||||
result,
|
|
||||||
expected_status="rolled-back",
|
|
||||||
checks={
|
|
||||||
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
|
||||||
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
|
||||||
database_url, database = _sqlite_database(root)
|
|
||||||
available = {"retirement-example": _retirement_failure_manifest()}
|
|
||||||
with database.session() as session:
|
|
||||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
|
||||||
plan = save_module_install_plan(session, [{
|
|
||||||
"module_id": "retirement-example",
|
|
||||||
"action": "uninstall",
|
|
||||||
"python_package": "govoplan-retirement-example",
|
|
||||||
"destroy_data": True,
|
|
||||||
}])
|
|
||||||
session.commit()
|
|
||||||
with _patch_subprocess(_success_run):
|
|
||||||
result = run_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available,
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url=database_url,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
)
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
return _scenario_result(
|
|
||||||
"destructive-retirement-failure",
|
|
||||||
root,
|
|
||||||
result,
|
|
||||||
expected_status="rolled-back",
|
|
||||||
checks={
|
|
||||||
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
|
||||||
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def postgres_hooks(root: Path) -> dict[str, object]:
|
|
||||||
_, database = _sqlite_database(root)
|
|
||||||
backup_script = (
|
|
||||||
"import json, os, pathlib; "
|
|
||||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
|
||||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
|
||||||
)
|
|
||||||
check_script = (
|
|
||||||
"import os, pathlib; "
|
|
||||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
|
||||||
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
|
||||||
)
|
|
||||||
restore_script = (
|
|
||||||
"import os, pathlib; "
|
|
||||||
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
|
||||||
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
|
||||||
)
|
|
||||||
with database.session() as session:
|
|
||||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
|
||||||
plan = save_module_install_plan(session, [{
|
|
||||||
"module_id": "postgres-hook-example",
|
|
||||||
"action": "install",
|
|
||||||
"python_package": "govoplan-postgres-hook-example",
|
|
||||||
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
|
||||||
}])
|
|
||||||
session.commit()
|
|
||||||
result = run_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available_module_manifests(),
|
|
||||||
current_enabled=("tenancy", "access"),
|
|
||||||
desired_enabled=("tenancy", "access"),
|
|
||||||
database_url="postgresql://db.example.invalid/govoplan",
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
migrate_database=True,
|
|
||||||
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
|
||||||
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
|
||||||
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
|
||||||
dry_run=True,
|
|
||||||
)
|
|
||||||
with _patch_subprocess(_non_shell_success_run):
|
|
||||||
rollback = rollback_module_install_run(
|
|
||||||
run_id=result.run_id,
|
|
||||||
runtime_dir=root / "installer",
|
|
||||||
database_url="postgresql://db.example.invalid/govoplan",
|
|
||||||
)
|
|
||||||
run_dir = result.record_path.parent
|
|
||||||
record = _run_record(root, result.run_id)
|
|
||||||
ok = (
|
|
||||||
result.status == "dry-run"
|
|
||||||
and rollback.status == "rolled-back"
|
|
||||||
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
|
||||||
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
|
||||||
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"scenario": "postgres-hooks",
|
|
||||||
"ok": ok,
|
|
||||||
"root": str(root),
|
|
||||||
"run_id": result.run_id,
|
|
||||||
"record_path": str(result.record_path),
|
|
||||||
"rollback_status": rollback.status,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def queue_and_lock(root: Path) -> dict[str, object]:
|
|
||||||
runtime_dir = root / "installer"
|
|
||||||
status = update_module_installer_daemon_status(
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
patch={"status": "polling", "current_request_id": None},
|
|
||||||
)
|
|
||||||
request = queue_module_installer_request(
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
requested_by="drill",
|
|
||||||
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
|
||||||
)
|
|
||||||
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
|
||||||
assert claimed is not None
|
|
||||||
failed = update_module_installer_request(
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
request_id=str(claimed["request_id"]),
|
|
||||||
patch={"status": "failed", "error": "simulated drill failure"},
|
|
||||||
)
|
|
||||||
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
|
||||||
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
|
||||||
lock_path = runtime_dir / "install.lock"
|
|
||||||
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
|
||||||
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
|
||||||
lock_path.unlink()
|
|
||||||
return {
|
|
||||||
"scenario": "queue-and-lock",
|
|
||||||
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
|
||||||
"root": str(root),
|
|
||||||
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
|
||||||
database_url = f"sqlite:///{root / 'drill.db'}"
|
|
||||||
configure_database(database_url)
|
|
||||||
database = get_database()
|
|
||||||
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
|
||||||
return database_url, database
|
|
||||||
|
|
||||||
|
|
||||||
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
|
||||||
with database.session() as session:
|
|
||||||
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
|
||||||
plan = save_module_install_plan(session, [{
|
|
||||||
"module_id": module_id,
|
|
||||||
"action": "install",
|
|
||||||
"python_package": f"govoplan-{module_id}",
|
|
||||||
"python_ref": f"govoplan-{module_id}==0.1.0",
|
|
||||||
}])
|
|
||||||
save_desired_enabled_modules(session, ("tenancy", "access"))
|
|
||||||
session.commit()
|
|
||||||
return plan
|
|
||||||
|
|
||||||
|
|
||||||
def _retirement_failure_manifest() -> ModuleManifest:
|
|
||||||
def destroy(_session: object, _module_id: str) -> None:
|
|
||||||
raise RuntimeError("simulated destructive retirement failure")
|
|
||||||
|
|
||||||
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
|
||||||
return MigrationRetirementPlan(
|
|
||||||
supported=True,
|
|
||||||
summary="Drill retirement provider supports destructive cleanup.",
|
|
||||||
destroy_data_supported=True,
|
|
||||||
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
|
||||||
destroy_data_executor=destroy,
|
|
||||||
)
|
|
||||||
|
|
||||||
return ModuleManifest(
|
|
||||||
id="retirement-example",
|
|
||||||
name="Retirement example",
|
|
||||||
version="0.1.0",
|
|
||||||
migration_spec=MigrationSpec(
|
|
||||||
module_id="retirement-example",
|
|
||||||
retirement_supported=True,
|
|
||||||
retirement_provider=provider,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _scenario_result(
|
|
||||||
scenario: str,
|
|
||||||
root: Path,
|
|
||||||
result: Any,
|
|
||||||
*,
|
|
||||||
expected_status: str,
|
|
||||||
checks: dict[str, bool],
|
|
||||||
) -> dict[str, object]:
|
|
||||||
return {
|
|
||||||
"scenario": scenario,
|
|
||||||
"ok": result.status == expected_status and all(checks.values()),
|
|
||||||
"root": str(root),
|
|
||||||
"run_id": result.run_id,
|
|
||||||
"record_path": str(result.record_path),
|
|
||||||
"status": result.status,
|
|
||||||
"checks": checks,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
|
||||||
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
|
||||||
|
|
||||||
|
|
||||||
def _nested(value: dict[str, object], *keys: str) -> object:
|
|
||||||
current: object = value
|
|
||||||
for key in keys:
|
|
||||||
if not isinstance(current, dict):
|
|
||||||
return None
|
|
||||||
current = current.get(key)
|
|
||||||
return current
|
|
||||||
|
|
||||||
|
|
||||||
def _command_text(argv: object) -> str:
|
|
||||||
if isinstance(argv, list | tuple):
|
|
||||||
return " ".join(str(item) for item in argv)
|
|
||||||
return str(argv)
|
|
||||||
|
|
||||||
|
|
||||||
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
|
||||||
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
|
||||||
|
|
||||||
|
|
||||||
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
|
||||||
text = _command_text(argv)
|
|
||||||
if "pip freeze" in text:
|
|
||||||
return _completed(stdout="govoplan-core==0.0.0\n")
|
|
||||||
return _completed()
|
|
||||||
|
|
||||||
|
|
||||||
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
|
||||||
if kwargs.get("shell"):
|
|
||||||
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
|
||||||
return _success_run(argv, *_args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
|
||||||
text = _command_text(argv)
|
|
||||||
if "pip install" in text and "==" in text and "-r" not in text:
|
|
||||||
return _completed(23, stderr="simulated package install failure")
|
|
||||||
return _success_run(argv, *_args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
|
||||||
text = _command_text(argv)
|
|
||||||
if "govoplan_core.commands.init_db" in text:
|
|
||||||
return _completed(24, stderr="simulated migration failure")
|
|
||||||
return _success_run(argv, *_args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
|
||||||
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
|
||||||
return _completed(25, stderr="simulated restart failure")
|
|
||||||
return _success_run(argv, *_args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
|
||||||
import govoplan_core.core.module_installer as installer
|
|
||||||
|
|
||||||
original = installer.subprocess.run
|
|
||||||
installer.subprocess.run = fake_run # type: ignore[assignment]
|
|
||||||
try:
|
|
||||||
yield
|
|
||||||
finally:
|
|
||||||
installer.subprocess.run = original # type: ignore[assignment]
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _patch_urlopen() -> Iterable[None]:
|
|
||||||
import govoplan_core.core.module_installer as installer
|
|
||||||
|
|
||||||
original = installer.urllib.request.urlopen
|
|
||||||
|
|
||||||
def fail(*_args: object, **_kwargs: object) -> None:
|
|
||||||
raise URLError("simulated health timeout")
|
|
||||||
|
|
||||||
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
|
||||||
try:
|
|
||||||
yield
|
|
||||||
finally:
|
|
||||||
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
|
||||||
|
|
||||||
|
|
||||||
SCENARIOS: dict[str, Scenario] = {
|
|
||||||
"destructive-retirement-failure": destructive_retirement_failure,
|
|
||||||
"health-timeout": health_timeout,
|
|
||||||
"migration-failure": migration_failure,
|
|
||||||
"package-failure": package_failure,
|
|
||||||
"postgres-hooks": postgres_hooks,
|
|
||||||
"queue-and-lock": queue_and_lock,
|
|
||||||
"restart-failure": restart_failure,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import base64
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
SRC = ROOT / "src"
|
|
||||||
|
|
||||||
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
|
||||||
("core-only", "tenancy,access,admin"),
|
|
||||||
("files-only", "tenancy,access,admin,files"),
|
|
||||||
("mail-only", "tenancy,access,admin,mail"),
|
|
||||||
("campaign-only", "tenancy,access,admin,campaigns"),
|
|
||||||
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
|
||||||
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
|
||||||
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--database-url",
|
|
||||||
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
|
||||||
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--module-set",
|
|
||||||
action="append",
|
|
||||||
default=[],
|
|
||||||
metavar="NAME=MODULES",
|
|
||||||
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--reset-schema",
|
|
||||||
action="store_true",
|
|
||||||
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
|
||||||
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
|
||||||
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
|
||||||
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
database_url = (args.database_url or "").strip()
|
|
||||||
if not database_url:
|
|
||||||
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
|
||||||
if not database_url.startswith("postgresql"):
|
|
||||||
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
|
||||||
|
|
||||||
module_sets = _parse_module_sets(args.module_set)
|
|
||||||
ok = True
|
|
||||||
for name, modules in module_sets:
|
|
||||||
print(f"\n== {name}: {modules} ==")
|
|
||||||
if args.reset_schema:
|
|
||||||
_reset_public_schema(database_url)
|
|
||||||
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
|
||||||
if not args.skip_migrate:
|
|
||||||
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
|
||||||
if not args.skip_smoke:
|
|
||||||
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
|
||||||
return 0 if ok else 1
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
|
||||||
if not raw:
|
|
||||||
return DEFAULT_MODULE_SETS
|
|
||||||
parsed: list[tuple[str, str]] = []
|
|
||||||
for item in raw:
|
|
||||||
name, separator, modules = item.partition("=")
|
|
||||||
if not separator or not name.strip() or not modules.strip():
|
|
||||||
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
|
||||||
parsed.append((name.strip(), modules.strip()))
|
|
||||||
return tuple(parsed)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
|
||||||
env = os.environ.copy()
|
|
||||||
env["APP_ENV"] = app_env
|
|
||||||
env["DATABASE_URL"] = database_url
|
|
||||||
env["ENABLED_MODULES"] = modules
|
|
||||||
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
|
||||||
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
|
||||||
env["CELERY_ENABLED"] = "false"
|
|
||||||
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
|
||||||
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
|
||||||
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
|
||||||
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
|
||||||
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
|
||||||
return env
|
|
||||||
|
|
||||||
|
|
||||||
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
|
||||||
print("+ " + " ".join(command))
|
|
||||||
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
|
||||||
if result.returncode != 0:
|
|
||||||
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _reset_public_schema(database_url: str) -> None:
|
|
||||||
try:
|
|
||||||
from sqlalchemy import create_engine
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
|
||||||
|
|
||||||
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
|
||||||
try:
|
|
||||||
with engine.connect() as connection:
|
|
||||||
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
|
||||||
connection.exec_driver_sql("CREATE SCHEMA public")
|
|
||||||
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
|
||||||
finally:
|
|
||||||
engine.dispose()
|
|
||||||
print("Reset PostgreSQL schema: public")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PROFILE_ROOT="$ROOT/dev/production-like"
|
|
||||||
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
|
||||||
if [ ! -f "$PROFILE_ENV" ]; then
|
|
||||||
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
|
||||||
fi
|
|
||||||
|
|
||||||
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
|
||||||
DOCKER="${DOCKER:-docker}"
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'EOF'
|
|
||||||
Usage: scripts/production-like-dev.sh <command> [--yes]
|
|
||||||
|
|
||||||
Commands:
|
|
||||||
start Start the production-like API, worker, WebUI, PostgreSQL, and Redis profile.
|
|
||||||
stop Stop PostgreSQL and Redis profile containers.
|
|
||||||
status Show profile container status and validate the effective environment.
|
|
||||||
seed Run migrations and seed development data against the profile database.
|
|
||||||
reset --yes Stop containers, remove profile volumes, and remove runtime/production-like data.
|
|
||||||
validate-config Validate the profile environment only.
|
|
||||||
|
|
||||||
The start command delegates to scripts/launch-production-like-dev.sh. Stop/reset
|
|
||||||
operate on Docker dependencies; API/WebUI/worker processes started by the
|
|
||||||
launcher are stopped with Ctrl+C in that launcher terminal.
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
printf 'production-like-dev: %s\n' "$*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
command="${1:-}"
|
|
||||||
if [ -z "$command" ] || [ "$command" = "-h" ] || [ "$command" = "--help" ]; then
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
shift || true
|
|
||||||
|
|
||||||
yes=0
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in
|
|
||||||
--yes) yes=1 ;;
|
|
||||||
*) fail "unknown argument: $arg" ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
|
||||||
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
|
||||||
|
|
||||||
set -a
|
|
||||||
# shellcheck disable=SC1090
|
|
||||||
. "$PROFILE_ENV"
|
|
||||||
set +a
|
|
||||||
|
|
||||||
export APP_ENV="${APP_ENV:-staging}"
|
|
||||||
export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
|
|
||||||
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
|
||||||
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
|
||||||
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
|
||||||
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
|
||||||
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
|
||||||
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
|
||||||
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
|
||||||
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
|
||||||
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
|
||||||
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}"
|
|
||||||
|
|
||||||
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
|
||||||
MASTER_KEY_B64="$("$PYTHON" -m govoplan_core.commands.config env-template --profile production-like --generate-secrets | sed -n 's/^MASTER_KEY_B64=//p' | head -n 1)"
|
|
||||||
export MASTER_KEY_B64
|
|
||||||
fi
|
|
||||||
|
|
||||||
compose() {
|
|
||||||
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
validate_config() {
|
|
||||||
"$PYTHON" -m govoplan_core.commands.config validate --profile production-like
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$command" in
|
|
||||||
start)
|
|
||||||
exec "$ROOT/scripts/launch-production-like-dev.sh"
|
|
||||||
;;
|
|
||||||
stop)
|
|
||||||
compose down
|
|
||||||
;;
|
|
||||||
status)
|
|
||||||
compose ps
|
|
||||||
validate_config
|
|
||||||
;;
|
|
||||||
seed)
|
|
||||||
validate_config
|
|
||||||
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
|
||||||
;;
|
|
||||||
reset)
|
|
||||||
[ "$yes" = "1" ] || fail "reset is destructive; rerun with --yes"
|
|
||||||
compose down -v
|
|
||||||
rm -rf "$ROOT/runtime/production-like"
|
|
||||||
;;
|
|
||||||
validate-config)
|
|
||||||
validate_config
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
usage >&2
|
|
||||||
fail "unknown command: $command"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
@@ -1,259 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'USAGE'
|
|
||||||
Usage:
|
|
||||||
scripts/publish-release-catalog.sh --version <x.y.z> --catalog-signing-key <key-id=/path/private.pem> [options]
|
|
||||||
|
|
||||||
Generates a signed module package catalog for a GovOPlaN release, writes it into
|
|
||||||
govoplan-web/public/catalogs/v1, validates it, optionally builds the website,
|
|
||||||
and optionally commits/tags/pushes govoplan-web.
|
|
||||||
|
|
||||||
Options:
|
|
||||||
--version <x.y.z> Release version without leading v.
|
|
||||||
--channel <name> Catalog channel. Defaults to stable.
|
|
||||||
--sequence <number> Monotonic channel sequence. Defaults to UTC timestamp.
|
|
||||||
--expires-days <days> Catalog expiry window. Defaults to 90.
|
|
||||||
--catalog-signing-key <key-id=/path/private.pem>
|
|
||||||
Ed25519 private key. May be repeated for rotation.
|
|
||||||
--core-root <path> govoplan-core checkout. Defaults to this script parent.
|
|
||||||
--web-root <path> govoplan-web checkout. Defaults to ../govoplan-web.
|
|
||||||
--public-base-url <url> Public website URL. Defaults to https://govoplan.add-ideas.de.
|
|
||||||
--npm <path> npm executable used for optional web build.
|
|
||||||
--build-web Run npm run build in govoplan-web after generating files.
|
|
||||||
--commit Commit generated catalog files in govoplan-web.
|
|
||||||
--tag Tag govoplan-web with catalog-v<x.y.z>. Implies --commit.
|
|
||||||
--push Push govoplan-web branch and tag. Implies --commit.
|
|
||||||
--remote <name> Git remote for push. Defaults to origin.
|
|
||||||
--branch <name> Branch to push. Defaults to current govoplan-web branch.
|
|
||||||
-n, --dry-run Print commands and validate inputs without writing git changes.
|
|
||||||
-h, --help Show this help.
|
|
||||||
USAGE
|
|
||||||
}
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
echo "error: $*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PARENT="$(dirname "$ROOT")"
|
|
||||||
CORE_ROOT="$ROOT"
|
|
||||||
WEB_ROOT="$PARENT/govoplan-web"
|
|
||||||
VERSION=""
|
|
||||||
CHANNEL="stable"
|
|
||||||
SEQUENCE=""
|
|
||||||
EXPIRES_DAYS="90"
|
|
||||||
PUBLIC_BASE_URL="https://govoplan.add-ideas.de"
|
|
||||||
REMOTE="origin"
|
|
||||||
BRANCH=""
|
|
||||||
BUILD_WEB=0
|
|
||||||
COMMIT=0
|
|
||||||
TAG_WEB=0
|
|
||||||
PUSH=0
|
|
||||||
DRY_RUN=0
|
|
||||||
SIGNING_KEYS=()
|
|
||||||
NPM_BIN="${NPM:-}"
|
|
||||||
|
|
||||||
if [[ -z "${PYTHON:-}" ]]; then
|
|
||||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
|
||||||
PYTHON="$ROOT/.venv/bin/python"
|
|
||||||
else
|
|
||||||
PYTHON="python3"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "$NPM_BIN" ]]; then
|
|
||||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
|
||||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
|
||||||
else
|
|
||||||
NPM_BIN="npm"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case "$1" in
|
|
||||||
--version)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
VERSION="${2#v}"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--channel)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CHANNEL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--sequence)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
SEQUENCE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--expires-days)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
EXPIRES_DAYS="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-signing-key)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
SIGNING_KEYS+=("$2")
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--core-root)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CORE_ROOT="$(cd "$2" && pwd)"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--web-root)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
WEB_ROOT="$(cd "$2" && pwd)"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--public-base-url)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
PUBLIC_BASE_URL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--npm)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
NPM_BIN="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--build-web)
|
|
||||||
BUILD_WEB=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--commit)
|
|
||||||
COMMIT=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--tag)
|
|
||||||
TAG_WEB=1
|
|
||||||
COMMIT=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--push)
|
|
||||||
PUSH=1
|
|
||||||
COMMIT=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--remote)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
REMOTE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--branch)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
BRANCH="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-n|--dry-run)
|
|
||||||
DRY_RUN=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown argument: $1" >&2
|
|
||||||
usage >&2
|
|
||||||
exit 2
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
[[ -n "$VERSION" ]] || fail "--version is required"
|
|
||||||
[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION"
|
|
||||||
[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required"
|
|
||||||
[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT"
|
|
||||||
[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT"
|
|
||||||
command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON"
|
|
||||||
|
|
||||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
|
||||||
command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "$BRANCH" ]]; then
|
|
||||||
BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)"
|
|
||||||
fi
|
|
||||||
[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch"
|
|
||||||
|
|
||||||
CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json"
|
|
||||||
KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json"
|
|
||||||
TAG_NAME="catalog-v$VERSION"
|
|
||||||
|
|
||||||
run() {
|
|
||||||
printf '+'
|
|
||||||
printf ' %q' "$@"
|
|
||||||
printf '\n'
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
"$@"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
GEN_ARGS=(
|
|
||||||
"$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py"
|
|
||||||
--version "$VERSION"
|
|
||||||
--channel "$CHANNEL"
|
|
||||||
--expires-days "$EXPIRES_DAYS"
|
|
||||||
--catalog-output "$CATALOG_PATH"
|
|
||||||
--keyring-output "$KEYRING_PATH"
|
|
||||||
--public-base-url "$PUBLIC_BASE_URL"
|
|
||||||
)
|
|
||||||
if [[ -n "$SEQUENCE" ]]; then
|
|
||||||
GEN_ARGS+=(--sequence "$SEQUENCE")
|
|
||||||
fi
|
|
||||||
for signing_key in "${SIGNING_KEYS[@]}"; do
|
|
||||||
GEN_ARGS+=(--catalog-signing-key "$signing_key")
|
|
||||||
done
|
|
||||||
|
|
||||||
run "${GEN_ARGS[@]}"
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \
|
|
||||||
"$PYTHON" -m govoplan_core.commands.module_installer \
|
|
||||||
--validate-package-catalog "$CATALOG_PATH" \
|
|
||||||
--require-signed-catalog \
|
|
||||||
--approved-catalog-channel "$CHANNEL" \
|
|
||||||
--format json >/dev/null
|
|
||||||
else
|
|
||||||
echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$BUILD_WEB" -eq 1 ]]; then
|
|
||||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$COMMIT" -eq 1 ]]; then
|
|
||||||
run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH"
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
if git -C "$WEB_ROOT" diff --cached --quiet; then
|
|
||||||
echo "No govoplan-web catalog changes to commit."
|
|
||||||
else
|
|
||||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
|
||||||
if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then
|
|
||||||
fail "govoplan-web tag already exists: $TAG_NAME"
|
|
||||||
fi
|
|
||||||
run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$PUSH" -eq 1 ]]; then
|
|
||||||
if [[ "$TAG_WEB" -eq 1 ]]; then
|
|
||||||
run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME"
|
|
||||||
else
|
|
||||||
run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Catalog ready:"
|
|
||||||
echo " $CATALOG_PATH"
|
|
||||||
echo " $KEYRING_PATH"
|
|
||||||
echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json"
|
|
||||||
@@ -1,861 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'USAGE'
|
|
||||||
Usage:
|
|
||||||
scripts/push-release-tag.sh [options]
|
|
||||||
|
|
||||||
Bumps installable GovOPlaN package versions, commits pending changes in the
|
|
||||||
GovOPlaN release/module repositories, creates an annotated release tag in each
|
|
||||||
repo, and pushes branch+tag.
|
|
||||||
|
|
||||||
By default the script asks which version part to bump:
|
|
||||||
major X.Y.Z -> (X+1).0.0
|
|
||||||
minor X.Y.Z -> X.(Y+1).0
|
|
||||||
subversion X.Y.Z -> X.Y.(Z+1)
|
|
||||||
|
|
||||||
Options:
|
|
||||||
--bump <kind> Version bump: major, minor, subversion, or patch.
|
|
||||||
--version <x.y.z> Explicit target version instead of a bump. May equal
|
|
||||||
the current repo versions when they were bumped already.
|
|
||||||
-r, --remote <name> Remote to push to. Defaults to origin.
|
|
||||||
-b, --branch <name> Branch to update in every repo. Defaults to each current branch.
|
|
||||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
|
||||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
|
||||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
|
||||||
--warn-migration-audit Run the migration release audit without baseline strictness.
|
|
||||||
--strict-migration-audit
|
|
||||||
Require current Alembic heads to be recorded in the
|
|
||||||
latest migration release baseline.
|
|
||||||
--skip-migration-audit Do not run the release migration audit.
|
|
||||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
|
||||||
catalog into govoplan-web.
|
|
||||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
|
||||||
Defaults to ../govoplan-web.
|
|
||||||
--catalog-signing-key <key-id=/path/private.pem>
|
|
||||||
Catalog signing key for --publish-web-catalog. May be
|
|
||||||
repeated during key rotation.
|
|
||||||
--catalog-channel <name>
|
|
||||||
Catalog channel. Defaults to stable.
|
|
||||||
--catalog-sequence <n> Monotonic catalog sequence. Defaults to UTC timestamp.
|
|
||||||
--catalog-expires-days <days>
|
|
||||||
Catalog expiry window. Defaults to 90.
|
|
||||||
--catalog-public-url <url>
|
|
||||||
Public website URL. Defaults to
|
|
||||||
https://govoplan.add-ideas.de.
|
|
||||||
--build-web-catalog Run npm run build in govoplan-web before committing the
|
|
||||||
catalog update.
|
|
||||||
--npm <path> npm executable for lockfile generation.
|
|
||||||
-y, --yes Do not prompt before committing, tagging, and pushing.
|
|
||||||
-n, --dry-run Print intended actions without changing files or git state.
|
|
||||||
-h, --help Show this help.
|
|
||||||
|
|
||||||
Repos:
|
|
||||||
Installable release packages:
|
|
||||||
govoplan-access
|
|
||||||
govoplan-admin
|
|
||||||
govoplan-tenancy
|
|
||||||
govoplan-organizations
|
|
||||||
govoplan-identity
|
|
||||||
govoplan-idm
|
|
||||||
govoplan-policy
|
|
||||||
govoplan-audit
|
|
||||||
govoplan-dashboard
|
|
||||||
govoplan-files
|
|
||||||
govoplan-mail
|
|
||||||
govoplan-campaign
|
|
||||||
govoplan-calendar
|
|
||||||
govoplan-ops
|
|
||||||
govoplan-core
|
|
||||||
|
|
||||||
Roadmap/scaffold module repositories are tag-only until they have package
|
|
||||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
|
||||||
fit-connect, forms, identity-trust, ledger, notifications, payments,
|
|
||||||
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
|
||||||
xrechnung, and xta-osci.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
scripts/push-release-tag.sh
|
|
||||||
scripts/push-release-tag.sh --bump subversion
|
|
||||||
scripts/push-release-tag.sh --version 0.2.0 --message "Release v0.2.0"
|
|
||||||
USAGE
|
|
||||||
}
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
echo "error: $*" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
||||||
PARENT="$(dirname "$ROOT")"
|
|
||||||
PACKAGE_MODULE_REPOS=(
|
|
||||||
"$PARENT/govoplan-access"
|
|
||||||
"$PARENT/govoplan-admin"
|
|
||||||
"$PARENT/govoplan-tenancy"
|
|
||||||
"$PARENT/govoplan-organizations"
|
|
||||||
"$PARENT/govoplan-identity"
|
|
||||||
"$PARENT/govoplan-idm"
|
|
||||||
"$PARENT/govoplan-policy"
|
|
||||||
"$PARENT/govoplan-audit"
|
|
||||||
"$PARENT/govoplan-dashboard"
|
|
||||||
"$PARENT/govoplan-files"
|
|
||||||
"$PARENT/govoplan-mail"
|
|
||||||
"$PARENT/govoplan-campaign"
|
|
||||||
"$PARENT/govoplan-calendar"
|
|
||||||
"$PARENT/govoplan-docs"
|
|
||||||
"$PARENT/govoplan-ops"
|
|
||||||
)
|
|
||||||
TAG_ONLY_MODULE_REPOS=(
|
|
||||||
"$PARENT/govoplan-addresses"
|
|
||||||
"$PARENT/govoplan-appointments"
|
|
||||||
"$PARENT/govoplan-cases"
|
|
||||||
"$PARENT/govoplan-connectors"
|
|
||||||
"$PARENT/govoplan-dms"
|
|
||||||
"$PARENT/govoplan-erp"
|
|
||||||
"$PARENT/govoplan-fit-connect"
|
|
||||||
"$PARENT/govoplan-forms"
|
|
||||||
"$PARENT/govoplan-identity-trust"
|
|
||||||
"$PARENT/govoplan-ledger"
|
|
||||||
"$PARENT/govoplan-notifications"
|
|
||||||
"$PARENT/govoplan-payments"
|
|
||||||
"$PARENT/govoplan-portal"
|
|
||||||
"$PARENT/govoplan-reporting"
|
|
||||||
"$PARENT/govoplan-scheduling"
|
|
||||||
"$PARENT/govoplan-search"
|
|
||||||
"$PARENT/govoplan-tasks"
|
|
||||||
"$PARENT/govoplan-templates"
|
|
||||||
"$PARENT/govoplan-workflow"
|
|
||||||
"$PARENT/govoplan-xoev"
|
|
||||||
"$PARENT/govoplan-xrechnung"
|
|
||||||
"$PARENT/govoplan-xta-osci"
|
|
||||||
)
|
|
||||||
MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}")
|
|
||||||
PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT")
|
|
||||||
REPOS=(
|
|
||||||
"${MODULE_REPOS[@]}"
|
|
||||||
"$ROOT"
|
|
||||||
)
|
|
||||||
|
|
||||||
REMOTE="origin"
|
|
||||||
BRANCH=""
|
|
||||||
BUMP=""
|
|
||||||
TARGET_VERSION=""
|
|
||||||
COMMIT_MESSAGE=""
|
|
||||||
TAG_MESSAGE=""
|
|
||||||
DRY_RUN=0
|
|
||||||
YES=0
|
|
||||||
GENERATE_RELEASE_LOCK=1
|
|
||||||
MIGRATION_AUDIT_MODE="auto"
|
|
||||||
NPM_BIN="${NPM:-}"
|
|
||||||
PUBLISH_WEB_CATALOG=0
|
|
||||||
WEB_ROOT="$PARENT/govoplan-web"
|
|
||||||
CATALOG_CHANNEL="stable"
|
|
||||||
CATALOG_SEQUENCE=""
|
|
||||||
CATALOG_EXPIRES_DAYS="90"
|
|
||||||
CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de"
|
|
||||||
BUILD_WEB_CATALOG=0
|
|
||||||
CATALOG_SIGNING_KEYS=()
|
|
||||||
|
|
||||||
if [[ -z "${PYTHON:-}" ]]; then
|
|
||||||
if [[ -x "$ROOT/.venv/bin/python" ]]; then
|
|
||||||
PYTHON="$ROOT/.venv/bin/python"
|
|
||||||
else
|
|
||||||
PYTHON="python3"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "$NPM_BIN" ]]; then
|
|
||||||
if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then
|
|
||||||
NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm"
|
|
||||||
else
|
|
||||||
NPM_BIN="npm"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
normalize_bump() {
|
|
||||||
local value="${1,,}"
|
|
||||||
case "$value" in
|
|
||||||
major|minor|subversion)
|
|
||||||
printf '%s\n' "$value"
|
|
||||||
;;
|
|
||||||
patch|sub|sub-version)
|
|
||||||
printf '%s\n' "subversion"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
validate_version() {
|
|
||||||
[[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
version_gt() {
|
|
||||||
local new_major new_minor new_patch old_major old_minor old_patch
|
|
||||||
IFS=. read -r new_major new_minor new_patch <<< "$1"
|
|
||||||
IFS=. read -r old_major old_minor old_patch <<< "$2"
|
|
||||||
|
|
||||||
(( new_major > old_major )) && return 0
|
|
||||||
(( new_major < old_major )) && return 1
|
|
||||||
(( new_minor > old_minor )) && return 0
|
|
||||||
(( new_minor < old_minor )) && return 1
|
|
||||||
(( new_patch > old_patch )) && return 0
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
bump_version() {
|
|
||||||
local version="$1"
|
|
||||||
local bump="$2"
|
|
||||||
local major minor patch
|
|
||||||
IFS=. read -r major minor patch <<< "$version"
|
|
||||||
|
|
||||||
case "$bump" in
|
|
||||||
major)
|
|
||||||
printf '%s.0.0\n' "$((major + 1))"
|
|
||||||
;;
|
|
||||||
minor)
|
|
||||||
printf '%s.%s.0\n' "$major" "$((minor + 1))"
|
|
||||||
;;
|
|
||||||
subversion)
|
|
||||||
printf '%s.%s.%s\n' "$major" "$minor" "$((patch + 1))"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
fail "unknown bump kind: $bump"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
get_project_name() {
|
|
||||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["name"])' "$1/pyproject.toml"
|
|
||||||
}
|
|
||||||
|
|
||||||
get_project_version() {
|
|
||||||
"$PYTHON" -c 'import pathlib, sys, tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$1/pyproject.toml"
|
|
||||||
}
|
|
||||||
|
|
||||||
update_pyproject() {
|
|
||||||
local repo="$1"
|
|
||||||
local version="$2"
|
|
||||||
|
|
||||||
"$PYTHON" - "$repo/pyproject.toml" "$version" <<'PYCODE'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import tomllib
|
|
||||||
|
|
||||||
path = pathlib.Path(sys.argv[1])
|
|
||||||
new_version = sys.argv[2]
|
|
||||||
text = path.read_text()
|
|
||||||
data = tomllib.loads(text)
|
|
||||||
project = data["project"]
|
|
||||||
name = project["name"]
|
|
||||||
old_version = project["version"]
|
|
||||||
|
|
||||||
text, version_count = re.subn(
|
|
||||||
r'(?m)^version\s*=\s*["\'][^"\']+["\']\s*$',
|
|
||||||
f'version = "{new_version}"',
|
|
||||||
text,
|
|
||||||
count=1,
|
|
||||||
)
|
|
||||||
if version_count != 1:
|
|
||||||
raise SystemExit(f"could not update project.version in {path}")
|
|
||||||
|
|
||||||
if name != "govoplan-core":
|
|
||||||
text, dependency_count = re.subn(
|
|
||||||
r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+',
|
|
||||||
rf'\g<1>{new_version}',
|
|
||||||
text,
|
|
||||||
)
|
|
||||||
if dependency_count < 1:
|
|
||||||
raise SystemExit(f"could not update govoplan-core dependency in {path}")
|
|
||||||
|
|
||||||
path.write_text(text)
|
|
||||||
print(f"{name}: {old_version} -> {new_version}")
|
|
||||||
PYCODE
|
|
||||||
}
|
|
||||||
|
|
||||||
update_manifest_version() {
|
|
||||||
local repo="$1"
|
|
||||||
local project_name="$2"
|
|
||||||
local version="$3"
|
|
||||||
local manifest_path=""
|
|
||||||
|
|
||||||
case "$project_name" in
|
|
||||||
govoplan-core)
|
|
||||||
return 0
|
|
||||||
;;
|
|
||||||
govoplan-access)
|
|
||||||
manifest_path="$repo/src/govoplan_access/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-admin)
|
|
||||||
manifest_path="$repo/src/govoplan_admin/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-tenancy)
|
|
||||||
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-organizations)
|
|
||||||
manifest_path="$repo/src/govoplan_organizations/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-identity)
|
|
||||||
manifest_path="$repo/src/govoplan_identity/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-idm)
|
|
||||||
manifest_path="$repo/src/govoplan_idm/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-policy)
|
|
||||||
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-audit)
|
|
||||||
manifest_path="$repo/src/govoplan_audit/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-files)
|
|
||||||
manifest_path="$repo/src/govoplan_files/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-mail)
|
|
||||||
manifest_path="$repo/src/govoplan_mail/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-campaign)
|
|
||||||
manifest_path="$repo/src/govoplan_campaign/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
govoplan-calendar)
|
|
||||||
manifest_path="$repo/src/govoplan_calendar/backend/manifest.py"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
fail "unknown project for manifest version update: $project_name"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
[[ -f "$manifest_path" ]] || fail "missing module manifest: $manifest_path"
|
|
||||||
|
|
||||||
"$PYTHON" - "$manifest_path" "$version" <<'PYCODE'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
path = pathlib.Path(sys.argv[1])
|
|
||||||
new_version = sys.argv[2]
|
|
||||||
text = path.read_text()
|
|
||||||
text, count = re.subn(
|
|
||||||
r'(?m)^(\s*version=)["\'][^"\']+["\'](,?\s*)$',
|
|
||||||
rf'\1"{new_version}"\2',
|
|
||||||
text,
|
|
||||||
count=1,
|
|
||||||
)
|
|
||||||
if count != 1:
|
|
||||||
raise SystemExit(f"could not update ModuleManifest.version in {path}")
|
|
||||||
path.write_text(text)
|
|
||||||
PYCODE
|
|
||||||
}
|
|
||||||
|
|
||||||
update_webui_package() {
|
|
||||||
local repo="$1"
|
|
||||||
local project_name="$2"
|
|
||||||
local version="$3"
|
|
||||||
local package_path=""
|
|
||||||
local release_package_path="$repo/webui/package.release.json"
|
|
||||||
|
|
||||||
for package_path in "$repo/package.json" "$repo/webui/package.json"; do
|
|
||||||
[[ -f "$package_path" ]] || continue
|
|
||||||
"$PYTHON" - "$package_path" "$project_name" "$version" <<'PYCODE'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
|
|
||||||
path = pathlib.Path(sys.argv[1])
|
|
||||||
project_name = sys.argv[2]
|
|
||||||
new_version = sys.argv[3]
|
|
||||||
|
|
||||||
data = json.loads(path.read_text())
|
|
||||||
data["version"] = new_version
|
|
||||||
if project_name != "govoplan-core":
|
|
||||||
peers = data.setdefault("peerDependencies", {})
|
|
||||||
if "@govoplan/core-webui" in peers:
|
|
||||||
peers["@govoplan/core-webui"] = f"^{new_version}"
|
|
||||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
|
||||||
PYCODE
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$project_name" == "govoplan-core" && -f "$release_package_path" ]]; then
|
|
||||||
"$PYTHON" - "$release_package_path" "$version" <<'PYCODE'
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
path = pathlib.Path(sys.argv[1])
|
|
||||||
new_version = sys.argv[2]
|
|
||||||
|
|
||||||
data = json.loads(path.read_text())
|
|
||||||
data["version"] = new_version
|
|
||||||
dependencies = data.setdefault("dependencies", {})
|
|
||||||
for name, spec in list(dependencies.items()):
|
|
||||||
if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"):
|
|
||||||
dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec)
|
|
||||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
|
||||||
PYCODE
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
update_version_files() {
|
|
||||||
local repo="$1"
|
|
||||||
local version="$2"
|
|
||||||
local project_name="${PROJECT_NAMES[$repo]}"
|
|
||||||
|
|
||||||
update_pyproject "$repo" "$version"
|
|
||||||
update_manifest_version "$repo" "$project_name" "$version"
|
|
||||||
update_webui_package "$repo" "$project_name" "$version"
|
|
||||||
}
|
|
||||||
|
|
||||||
refresh_development_webui_lock() {
|
|
||||||
[[ -f "$ROOT/webui/package.json" ]] || return 0
|
|
||||||
[[ -f "$ROOT/webui/package-lock.json" ]] || return 0
|
|
||||||
|
|
||||||
run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$ROOT/webui" install --package-lock-only --ignore-scripts
|
|
||||||
}
|
|
||||||
|
|
||||||
generate_release_lock() {
|
|
||||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 0 ]]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
[[ -x "$ROOT/scripts/generate-release-lock.sh" ]] || fail "missing executable release lock generator: $ROOT/scripts/generate-release-lock.sh"
|
|
||||||
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
|
||||||
}
|
|
||||||
|
|
||||||
run_migration_release_audit() {
|
|
||||||
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
|
||||||
local command=("$PYTHON" "$audit_script")
|
|
||||||
|
|
||||||
case "$MIGRATION_AUDIT_MODE" in
|
|
||||||
skip)
|
|
||||||
echo "Migration release audit: skipped"
|
|
||||||
return
|
|
||||||
;;
|
|
||||||
strict)
|
|
||||||
command+=("--strict")
|
|
||||||
;;
|
|
||||||
auto)
|
|
||||||
command+=("--strict-if-baseline")
|
|
||||||
;;
|
|
||||||
warn)
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
|
||||||
run "${command[@]}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print_command() {
|
|
||||||
printf '+'
|
|
||||||
printf ' %q' "$@"
|
|
||||||
printf '\n'
|
|
||||||
}
|
|
||||||
|
|
||||||
run() {
|
|
||||||
print_command "$@"
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
"$@"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
prompt_for_bump() {
|
|
||||||
local answer=""
|
|
||||||
|
|
||||||
if [[ -n "$BUMP" || -n "$TARGET_VERSION" ]]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -t 0 ]]; then
|
|
||||||
fail "pass --bump major|minor|subversion or --version x.y.z for non-interactive use"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Select version increase:"
|
|
||||||
echo " 1) major X.Y.Z -> (X+1).0.0"
|
|
||||||
echo " 2) minor X.Y.Z -> X.(Y+1).0"
|
|
||||||
echo " 3) subversion X.Y.Z -> X.Y.(Z+1)"
|
|
||||||
printf 'Choice [subversion]: '
|
|
||||||
read -r answer
|
|
||||||
|
|
||||||
case "${answer,,}" in
|
|
||||||
""|3|subversion|patch|sub|sub-version)
|
|
||||||
BUMP="subversion"
|
|
||||||
;;
|
|
||||||
1|major)
|
|
||||||
BUMP="major"
|
|
||||||
;;
|
|
||||||
2|minor)
|
|
||||||
BUMP="minor"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
fail "unknown version bump: $answer"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
confirm_release() {
|
|
||||||
local answer=""
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 1 || "$YES" -eq 1 ]]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -t 0 ]]; then
|
|
||||||
fail "refusing to commit, tag, and push without confirmation; pass --yes for non-interactive use"
|
|
||||||
fi
|
|
||||||
|
|
||||||
printf 'Commit all changes, tag all repos, and push to %s? [y/N] ' "$REMOTE"
|
|
||||||
read -r answer
|
|
||||||
case "${answer,,}" in
|
|
||||||
y|yes)
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Aborted."
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case "$1" in
|
|
||||||
--bump)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
BUMP="$(normalize_bump "$2")" || fail "unknown bump kind: $2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--version)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
TARGET_VERSION="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-r|--remote)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
REMOTE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-b|--branch)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
BRANCH="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-m|--message)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
COMMIT_MESSAGE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--tag-message)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
TAG_MESSAGE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--skip-release-lock)
|
|
||||||
GENERATE_RELEASE_LOCK=0
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--warn-migration-audit)
|
|
||||||
MIGRATION_AUDIT_MODE="warn"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--strict-migration-audit)
|
|
||||||
MIGRATION_AUDIT_MODE="strict"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--skip-migration-audit)
|
|
||||||
MIGRATION_AUDIT_MODE="skip"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--publish-web-catalog)
|
|
||||||
PUBLISH_WEB_CATALOG=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--web-root)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
WEB_ROOT="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-signing-key)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CATALOG_SIGNING_KEYS+=("$2")
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-channel)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CATALOG_CHANNEL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-sequence)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CATALOG_SEQUENCE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-expires-days)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CATALOG_EXPIRES_DAYS="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--catalog-public-url)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
CATALOG_PUBLIC_URL="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--build-web-catalog)
|
|
||||||
BUILD_WEB_CATALOG=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--npm)
|
|
||||||
[[ $# -ge 2 ]] || fail "missing value for $1"
|
|
||||||
NPM_BIN="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-y|--yes)
|
|
||||||
YES=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-n|--dry-run)
|
|
||||||
DRY_RUN=1
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "Unknown argument: $1" >&2
|
|
||||||
usage >&2
|
|
||||||
exit 2
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then
|
|
||||||
fail "use either --bump or --version, not both"
|
|
||||||
fi
|
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then
|
|
||||||
fail "--publish-web-catalog requires at least one --catalog-signing-key"
|
|
||||||
fi
|
|
||||||
|
|
||||||
prompt_for_bump
|
|
||||||
|
|
||||||
if ! command -v "$PYTHON" >/dev/null 2>&1; then
|
|
||||||
fail "Python interpreter not found: $PYTHON"
|
|
||||||
fi
|
|
||||||
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
|
|
||||||
fail "npm executable not found: $NPM_BIN"
|
|
||||||
fi
|
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
|
||||||
[[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT"
|
|
||||||
[[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh"
|
|
||||||
fi
|
|
||||||
|
|
||||||
declare -A PROJECT_NAMES
|
|
||||||
declare -A OLD_VERSIONS
|
|
||||||
declare -A BRANCHES
|
|
||||||
declare -A REPO_KINDS
|
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
|
||||||
[[ -d "$repo/.git" ]] || fail "not a git repo: $repo"
|
|
||||||
|
|
||||||
if [[ -f "$repo/pyproject.toml" ]]; then
|
|
||||||
REPO_KINDS["$repo"]="package"
|
|
||||||
PROJECT_NAMES["$repo"]="$(get_project_name "$repo")"
|
|
||||||
OLD_VERSIONS["$repo"]="$(get_project_version "$repo")"
|
|
||||||
|
|
||||||
validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo"
|
|
||||||
else
|
|
||||||
REPO_KINDS["$repo"]="tag-only"
|
|
||||||
PROJECT_NAMES["$repo"]="$(basename "$repo")"
|
|
||||||
OLD_VERSIONS["$repo"]="tag-only"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "$BRANCH" ]]; then
|
|
||||||
BRANCHES["$repo"]="$BRANCH"
|
|
||||||
else
|
|
||||||
BRANCHES["$repo"]="$(git -C "$repo" symbolic-ref --quiet --short HEAD || true)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
[[ -n "${BRANCHES[$repo]}" ]] || fail "cannot infer branch for $repo; pass --branch <name>"
|
|
||||||
git -C "$repo" check-ref-format "refs/heads/${BRANCHES[$repo]}" || fail "invalid branch name ${BRANCHES[$repo]} for $repo"
|
|
||||||
git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo"
|
|
||||||
|
|
||||||
upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)"
|
|
||||||
if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then
|
|
||||||
behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")"
|
|
||||||
[[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ -z "$TARGET_VERSION" ]]; then
|
|
||||||
BASE_VERSION="${OLD_VERSIONS[$ROOT]}"
|
|
||||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
|
||||||
if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then
|
|
||||||
fail "repo versions differ; pass --version x.y.z to choose an explicit target"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
TARGET_VERSION="$(bump_version "$BASE_VERSION" "$BUMP")"
|
|
||||||
fi
|
|
||||||
|
|
||||||
validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION"
|
|
||||||
|
|
||||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
|
||||||
if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if ! version_gt "$TARGET_VERSION" "${OLD_VERSIONS[$repo]}"; then
|
|
||||||
fail "target version $TARGET_VERSION must be greater than ${OLD_VERSIONS[$repo]} for ${PROJECT_NAMES[$repo]}"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
TAG="v$TARGET_VERSION"
|
|
||||||
COMMIT_MESSAGE="${COMMIT_MESSAGE:-Release $TAG}"
|
|
||||||
TAG_MESSAGE="${TAG_MESSAGE:-$COMMIT_MESSAGE}"
|
|
||||||
|
|
||||||
git -C "$ROOT" check-ref-format "refs/tags/$TAG" || fail "invalid tag name: $TAG"
|
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
|
||||||
if git -C "$repo" rev-parse --quiet --verify "refs/tags/$TAG" >/dev/null; then
|
|
||||||
fail "local tag already exists in ${PROJECT_NAMES[$repo]}: $TAG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
set +e
|
|
||||||
git -C "$repo" ls-remote --exit-code --tags "$REMOTE" "refs/tags/$TAG" >/dev/null
|
|
||||||
ls_remote_status=$?
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [[ "$ls_remote_status" -eq 0 ]]; then
|
|
||||||
fail "remote tag already exists for ${PROJECT_NAMES[$repo]} on $REMOTE: $TAG"
|
|
||||||
elif [[ "$ls_remote_status" -ne 2 ]]; then
|
|
||||||
fail "could not check remote tags for ${PROJECT_NAMES[$repo]} on $REMOTE"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "Release plan:"
|
|
||||||
echo " version: $TARGET_VERSION"
|
|
||||||
echo " tag: $TAG"
|
|
||||||
echo " remote: $REMOTE"
|
|
||||||
echo " commit message: $COMMIT_MESSAGE"
|
|
||||||
echo " package repos: ${#PACKAGE_REPOS[@]} versioned"
|
|
||||||
echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files"
|
|
||||||
if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
|
||||||
echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed"
|
|
||||||
else
|
|
||||||
echo " release lock: skipped"
|
|
||||||
fi
|
|
||||||
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
|
||||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
|
||||||
else
|
|
||||||
echo " web catalog: skipped"
|
|
||||||
fi
|
|
||||||
echo
|
|
||||||
|
|
||||||
for repo in "${REPOS[@]}"; do
|
|
||||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
|
||||||
echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}"
|
|
||||||
else
|
|
||||||
echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}"
|
|
||||||
fi
|
|
||||||
git -C "$repo" status --short
|
|
||||||
echo
|
|
||||||
done
|
|
||||||
|
|
||||||
run_migration_release_audit
|
|
||||||
|
|
||||||
confirm_release
|
|
||||||
|
|
||||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
||||||
echo "Would update version files in $repo to $TARGET_VERSION"
|
|
||||||
else
|
|
||||||
update_version_files "$repo" "$TARGET_VERSION"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
refresh_development_webui_lock
|
|
||||||
|
|
||||||
for repo in "${MODULE_REPOS[@]}"; do
|
|
||||||
run git -C "$repo" add -A
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
if git -C "$repo" diff --cached --quiet; then
|
|
||||||
if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then
|
|
||||||
fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump"
|
|
||||||
fi
|
|
||||||
echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD."
|
|
||||||
else
|
|
||||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
run git -C "$repo" commit -m "$COMMIT_MESSAGE"
|
|
||||||
fi
|
|
||||||
|
|
||||||
run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE"
|
|
||||||
done
|
|
||||||
|
|
||||||
for repo in "${MODULE_REPOS[@]}"; do
|
|
||||||
run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG"
|
|
||||||
done
|
|
||||||
|
|
||||||
generate_release_lock
|
|
||||||
|
|
||||||
run git -C "$ROOT" add -A
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 0 ]]; then
|
|
||||||
if git -C "$ROOT" diff --cached --quiet; then
|
|
||||||
fail "no staged changes for ${PROJECT_NAMES[$ROOT]} after version bump"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
run git -C "$ROOT" commit -m "$COMMIT_MESSAGE"
|
|
||||||
run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE"
|
|
||||||
run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG"
|
|
||||||
|
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
|
||||||
CATALOG_ARGS=(
|
|
||||||
"$ROOT/scripts/publish-release-catalog.sh"
|
|
||||||
--version "$TARGET_VERSION"
|
|
||||||
--channel "$CATALOG_CHANNEL"
|
|
||||||
--expires-days "$CATALOG_EXPIRES_DAYS"
|
|
||||||
--core-root "$ROOT"
|
|
||||||
--web-root "$WEB_ROOT"
|
|
||||||
--public-base-url "$CATALOG_PUBLIC_URL"
|
|
||||||
--npm "$NPM_BIN"
|
|
||||||
--commit
|
|
||||||
--tag
|
|
||||||
--push
|
|
||||||
--remote "$REMOTE"
|
|
||||||
)
|
|
||||||
if [[ -n "$BRANCH" ]]; then
|
|
||||||
CATALOG_ARGS+=(--branch "$BRANCH")
|
|
||||||
fi
|
|
||||||
if [[ -n "$CATALOG_SEQUENCE" ]]; then
|
|
||||||
CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE")
|
|
||||||
fi
|
|
||||||
if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then
|
|
||||||
CATALOG_ARGS+=(--build-web)
|
|
||||||
fi
|
|
||||||
for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do
|
|
||||||
CATALOG_ARGS+=(--catalog-signing-key "$signing_key")
|
|
||||||
done
|
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
||||||
CATALOG_ARGS+=(--dry-run)
|
|
||||||
fi
|
|
||||||
run "${CATALOG_ARGS[@]}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
||||||
echo "Dry run complete for $TAG across all GovOPlaN repos."
|
|
||||||
else
|
|
||||||
echo "Released $TAG across all GovOPlaN repos."
|
|
||||||
fi
|
|
||||||
@@ -1,422 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import ast
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class Migration:
|
|
||||||
owner: str
|
|
||||||
path: Path
|
|
||||||
revision: str
|
|
||||||
down_revisions: tuple[str, ...]
|
|
||||||
branch_labels: tuple[str, ...]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--workspace-root",
|
|
||||||
type=Path,
|
|
||||||
default=ROOT.parent,
|
|
||||||
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--baseline-file",
|
|
||||||
type=Path,
|
|
||||||
default=DEFAULT_BASELINE_FILE,
|
|
||||||
help="JSON file recording released migration heads.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--strict",
|
|
||||||
action="store_true",
|
|
||||||
help="Fail when current heads are not recorded in the latest release baseline.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--strict-if-baseline",
|
|
||||||
action="store_true",
|
|
||||||
help="Run in strict mode only after at least one release baseline is recorded.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--record-release",
|
|
||||||
metavar="VERSION",
|
|
||||||
help="Record the current reviewed migration heads as a release baseline.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--replace-release",
|
|
||||||
action="store_true",
|
|
||||||
help="Replace an existing release entry when used with --record-release.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--squash-plan",
|
|
||||||
action="store_true",
|
|
||||||
help="Print the reviewed/manual squash checklist for the current graph.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
migrations = discover_migrations(args.workspace_root)
|
|
||||||
baseline = load_baseline_file(args.baseline_file)
|
|
||||||
report = build_report(
|
|
||||||
migrations,
|
|
||||||
baseline=baseline,
|
|
||||||
baseline_file=args.baseline_file,
|
|
||||||
workspace_root=args.workspace_root,
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.record_release:
|
|
||||||
if report["graph_errors"]:
|
|
||||||
print_text_report(report)
|
|
||||||
return 1
|
|
||||||
baseline = record_release_baseline(
|
|
||||||
baseline,
|
|
||||||
report,
|
|
||||||
release=args.record_release,
|
|
||||||
replace=args.replace_release,
|
|
||||||
)
|
|
||||||
write_baseline_file(args.baseline_file, baseline)
|
|
||||||
report = build_report(
|
|
||||||
migrations,
|
|
||||||
baseline=baseline,
|
|
||||||
baseline_file=args.baseline_file,
|
|
||||||
workspace_root=args.workspace_root,
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.squash_plan:
|
|
||||||
print_squash_plan(report)
|
|
||||||
elif args.json:
|
|
||||||
print(json.dumps(report, indent=2, sort_keys=True))
|
|
||||||
else:
|
|
||||||
print_text_report(report)
|
|
||||||
|
|
||||||
has_graph_errors = bool(report["graph_errors"])
|
|
||||||
has_strict_errors = bool(report["strict_errors"])
|
|
||||||
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
|
||||||
if has_graph_errors or (strict_required and has_strict_errors):
|
|
||||||
return 1
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
|
||||||
migrations: list[Migration] = []
|
|
||||||
for versions_dir in _migration_version_dirs(workspace_root):
|
|
||||||
owner = owner_for_versions_dir(versions_dir)
|
|
||||||
for path in sorted(versions_dir.glob("*.py")):
|
|
||||||
if path.name == "__init__.py":
|
|
||||||
continue
|
|
||||||
migration = parse_migration_file(owner, path)
|
|
||||||
if migration is not None:
|
|
||||||
migrations.append(migration)
|
|
||||||
return migrations
|
|
||||||
|
|
||||||
|
|
||||||
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
|
||||||
candidates = [ROOT / "alembic" / "versions"]
|
|
||||||
for repo in sorted(workspace_root.glob("govoplan-*")):
|
|
||||||
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
|
||||||
seen: set[Path] = set()
|
|
||||||
result: list[Path] = []
|
|
||||||
for candidate in candidates:
|
|
||||||
resolved = candidate.resolve()
|
|
||||||
if resolved in seen or not candidate.is_dir():
|
|
||||||
continue
|
|
||||||
seen.add(resolved)
|
|
||||||
result.append(candidate)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def owner_for_versions_dir(versions_dir: Path) -> str:
|
|
||||||
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
|
||||||
return "govoplan-core"
|
|
||||||
for parent in versions_dir.parents:
|
|
||||||
if parent.name.startswith("govoplan-"):
|
|
||||||
return parent.name
|
|
||||||
return versions_dir.parent.name
|
|
||||||
|
|
||||||
|
|
||||||
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
|
||||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
|
||||||
values: dict[str, Any] = {}
|
|
||||||
for statement in tree.body:
|
|
||||||
if isinstance(statement, ast.Assign):
|
|
||||||
for target in statement.targets:
|
|
||||||
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
|
||||||
values[target.id] = ast.literal_eval(statement.value)
|
|
||||||
elif (
|
|
||||||
isinstance(statement, ast.AnnAssign)
|
|
||||||
and isinstance(statement.target, ast.Name)
|
|
||||||
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
|
||||||
and statement.value is not None
|
|
||||||
):
|
|
||||||
values[statement.target.id] = ast.literal_eval(statement.value)
|
|
||||||
revision = values.get("revision")
|
|
||||||
if not isinstance(revision, str):
|
|
||||||
return None
|
|
||||||
return Migration(
|
|
||||||
owner=owner,
|
|
||||||
path=path,
|
|
||||||
revision=revision,
|
|
||||||
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
|
||||||
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if isinstance(value, str):
|
|
||||||
return (value,)
|
|
||||||
if isinstance(value, (list, tuple, set)):
|
|
||||||
return tuple(str(item) for item in value if item is not None)
|
|
||||||
return (str(value),)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if isinstance(value, str):
|
|
||||||
return (value,)
|
|
||||||
if isinstance(value, (list, tuple, set)):
|
|
||||||
return tuple(str(item) for item in value)
|
|
||||||
return (str(value),)
|
|
||||||
|
|
||||||
|
|
||||||
def load_baseline_file(path: Path) -> dict[str, Any]:
|
|
||||||
if not path.exists():
|
|
||||||
return {"version": 1, "releases": [], "_missing": True}
|
|
||||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
raise SystemExit(f"{path} must contain a JSON object")
|
|
||||||
releases = payload.get("releases")
|
|
||||||
if not isinstance(releases, list):
|
|
||||||
raise SystemExit(f"{path} must contain a releases list")
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def build_report(
|
|
||||||
migrations: list[Migration],
|
|
||||||
*,
|
|
||||||
baseline: dict[str, Any],
|
|
||||||
baseline_file: Path,
|
|
||||||
workspace_root: Path,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
revisions: dict[str, Migration] = {}
|
|
||||||
graph_errors: list[str] = []
|
|
||||||
for migration in migrations:
|
|
||||||
existing = revisions.get(migration.revision)
|
|
||||||
if existing is not None:
|
|
||||||
graph_errors.append(
|
|
||||||
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
|
||||||
)
|
|
||||||
revisions[migration.revision] = migration
|
|
||||||
|
|
||||||
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
|
||||||
for migration in migrations:
|
|
||||||
for down_revision in migration.down_revisions:
|
|
||||||
if down_revision not in revisions:
|
|
||||||
graph_errors.append(
|
|
||||||
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
|
||||||
current_head_entries = [
|
|
||||||
{
|
|
||||||
"owner": revisions[revision].owner,
|
|
||||||
"revision": revision,
|
|
||||||
}
|
|
||||||
for revision in current_heads
|
|
||||||
]
|
|
||||||
owner_rows = []
|
|
||||||
for owner in sorted({migration.owner for migration in migrations}):
|
|
||||||
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
|
||||||
owner_revisions = {migration.revision for migration in owner_migrations}
|
|
||||||
owner_referenced = {
|
|
||||||
revision
|
|
||||||
for migration in owner_migrations
|
|
||||||
for revision in migration.down_revisions
|
|
||||||
if revision in owner_revisions
|
|
||||||
}
|
|
||||||
owner_heads = sorted(owner_revisions - owner_referenced)
|
|
||||||
owner_rows.append(
|
|
||||||
{
|
|
||||||
"owner": owner,
|
|
||||||
"migrations": len(owner_migrations),
|
|
||||||
"heads": owner_heads,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
releases = baseline.get("releases") or []
|
|
||||||
latest_release = releases[-1] if releases else None
|
|
||||||
latest_heads = _latest_release_heads(latest_release)
|
|
||||||
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
|
||||||
|
|
||||||
strict_errors: list[str] = []
|
|
||||||
if baseline.get("_missing"):
|
|
||||||
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
|
||||||
if not releases:
|
|
||||||
strict_errors.append("no released migration baseline has been recorded yet")
|
|
||||||
if unrecorded_heads:
|
|
||||||
strict_errors.append(
|
|
||||||
"current migration heads are not recorded in the latest release baseline: "
|
|
||||||
+ ", ".join(unrecorded_heads)
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"workspace_root": str(workspace_root),
|
|
||||||
"baseline_file": str(baseline_file),
|
|
||||||
"baseline_file_missing": bool(baseline.get("_missing")),
|
|
||||||
"release_count": len(releases),
|
|
||||||
"latest_release": latest_release,
|
|
||||||
"owners": owner_rows,
|
|
||||||
"current_heads": current_heads,
|
|
||||||
"current_head_entries": current_head_entries,
|
|
||||||
"graph_errors": graph_errors,
|
|
||||||
"strict_errors": strict_errors,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _latest_release_heads(latest_release: Any) -> set[str]:
|
|
||||||
if not isinstance(latest_release, dict):
|
|
||||||
return set()
|
|
||||||
heads = latest_release.get("heads") or []
|
|
||||||
graph_heads = latest_release.get("graph_heads") or []
|
|
||||||
result: set[str] = set()
|
|
||||||
for head_list in (heads, graph_heads):
|
|
||||||
if not isinstance(head_list, list):
|
|
||||||
continue
|
|
||||||
for entry in head_list:
|
|
||||||
if isinstance(entry, str):
|
|
||||||
result.add(entry)
|
|
||||||
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
|
||||||
result.add(entry["revision"])
|
|
||||||
owner_heads = latest_release.get("owner_heads") or []
|
|
||||||
if isinstance(owner_heads, list):
|
|
||||||
for entry in owner_heads:
|
|
||||||
if not isinstance(entry, dict):
|
|
||||||
continue
|
|
||||||
revisions = entry.get("revisions") or []
|
|
||||||
if isinstance(revisions, str):
|
|
||||||
result.add(revisions)
|
|
||||||
elif isinstance(revisions, list):
|
|
||||||
result.update(revision for revision in revisions if isinstance(revision, str))
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def record_release_baseline(
|
|
||||||
baseline: dict[str, Any],
|
|
||||||
report: dict[str, Any],
|
|
||||||
*,
|
|
||||||
release: str,
|
|
||||||
replace: bool,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
payload = {
|
|
||||||
key: value
|
|
||||||
for key, value in baseline.items()
|
|
||||||
if not key.startswith("_")
|
|
||||||
}
|
|
||||||
payload.setdefault("version", 1)
|
|
||||||
releases = list(payload.get("releases") or [])
|
|
||||||
existing_index = next(
|
|
||||||
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
if existing_index is not None and not replace:
|
|
||||||
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
|
||||||
|
|
||||||
entry = {
|
|
||||||
"release": release,
|
|
||||||
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
|
||||||
"squash_policy": "reviewed-manual",
|
|
||||||
"heads": report["current_head_entries"],
|
|
||||||
"owner_heads": [
|
|
||||||
{
|
|
||||||
"owner": owner["owner"],
|
|
||||||
"revisions": owner["heads"],
|
|
||||||
}
|
|
||||||
for owner in report["owners"]
|
|
||||||
],
|
|
||||||
}
|
|
||||||
if existing_index is None:
|
|
||||||
releases.append(entry)
|
|
||||||
else:
|
|
||||||
releases[existing_index] = entry
|
|
||||||
payload["releases"] = releases
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def print_text_report(report: dict[str, Any]) -> None:
|
|
||||||
print("Migration release audit")
|
|
||||||
print(f" workspace: {report['workspace_root']}")
|
|
||||||
print(f" baseline file: {report['baseline_file']}")
|
|
||||||
print(f" recorded releases: {report['release_count']}")
|
|
||||||
print()
|
|
||||||
print("Owners:")
|
|
||||||
for owner in report["owners"]:
|
|
||||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
|
||||||
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
|
||||||
print()
|
|
||||||
print("Current heads:")
|
|
||||||
for entry in report["current_head_entries"]:
|
|
||||||
print(f" {entry['owner']}: {entry['revision']}")
|
|
||||||
if not report["current_head_entries"]:
|
|
||||||
print(" -")
|
|
||||||
|
|
||||||
if report["graph_errors"]:
|
|
||||||
print()
|
|
||||||
print("Graph errors:")
|
|
||||||
for error in report["graph_errors"]:
|
|
||||||
print(f" - {error}")
|
|
||||||
|
|
||||||
if report["strict_errors"]:
|
|
||||||
print()
|
|
||||||
print("Release baseline warnings:")
|
|
||||||
for error in report["strict_errors"]:
|
|
||||||
print(f" - {error}")
|
|
||||||
|
|
||||||
|
|
||||||
def print_squash_plan(report: dict[str, Any]) -> None:
|
|
||||||
print("Manual migration squash plan")
|
|
||||||
print()
|
|
||||||
print("Policy:")
|
|
||||||
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
|
||||||
print(" - Squash only unreleased development migrations.")
|
|
||||||
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
|
||||||
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
|
||||||
print()
|
|
||||||
print("Current owner heads:")
|
|
||||||
for owner in report["owners"]:
|
|
||||||
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
|
||||||
print(f" - {owner['owner']}: {heads}")
|
|
||||||
print()
|
|
||||||
print("Current Alembic graph heads:")
|
|
||||||
for entry in report["current_head_entries"]:
|
|
||||||
print(f" - {entry['owner']}: {entry['revision']}")
|
|
||||||
if not report["current_head_entries"]:
|
|
||||||
print(" -")
|
|
||||||
print()
|
|
||||||
print("Release steps:")
|
|
||||||
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
|
||||||
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
|
||||||
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
|
||||||
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
|
||||||
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -94,6 +94,28 @@ class UserInfo(BaseModel):
|
|||||||
ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences)
|
ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences)
|
||||||
|
|
||||||
|
|
||||||
|
class AuthSessionUserInfo(BaseModel):
|
||||||
|
id: str
|
||||||
|
account_id: str
|
||||||
|
email: str
|
||||||
|
display_name: str | None = None
|
||||||
|
tenant_display_name: str | None = None
|
||||||
|
is_tenant_admin: bool = False
|
||||||
|
password_reset_required: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class AuthSessionResponse(BaseModel):
|
||||||
|
authenticated: bool = True
|
||||||
|
auth_method: Literal["session", "api_key"] = "session"
|
||||||
|
user: AuthSessionUserInfo
|
||||||
|
# Backwards-compatible alias for the active tenant.
|
||||||
|
tenant: TenantInfo
|
||||||
|
active_tenant: TenantInfo
|
||||||
|
session_id: str | None = None
|
||||||
|
api_key_id: str | None = None
|
||||||
|
expires_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
class ProfileUpdateRequest(BaseModel):
|
class ProfileUpdateRequest(BaseModel):
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
@@ -143,6 +165,40 @@ class PrincipalContextInfo(BaseModel):
|
|||||||
display_name: str | None = None
|
display_name: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AuthShellResponse(BaseModel):
|
||||||
|
user: UserInfo
|
||||||
|
# Backwards-compatible alias for the active tenant.
|
||||||
|
tenant: TenantInfo
|
||||||
|
active_tenant: TenantInfo
|
||||||
|
tenants: list[TenantMembershipInfo] = Field(default_factory=list)
|
||||||
|
scopes: list[str]
|
||||||
|
principal: PrincipalContextInfo | None = None
|
||||||
|
profile_loaded: bool = False
|
||||||
|
roles_loaded: bool = False
|
||||||
|
groups_loaded: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class AuthProfileResponse(BaseModel):
|
||||||
|
user: UserInfo
|
||||||
|
# Backwards-compatible alias for the active tenant.
|
||||||
|
tenant: TenantInfo
|
||||||
|
active_tenant: TenantInfo
|
||||||
|
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||||
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
|
default_language: str = "en"
|
||||||
|
profile_loaded: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class AuthRolesResponse(BaseModel):
|
||||||
|
roles: list[RoleInfo] = Field(default_factory=list)
|
||||||
|
roles_loaded: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class AuthGroupsResponse(BaseModel):
|
||||||
|
groups: list[GroupInfo] = Field(default_factory=list)
|
||||||
|
groups_loaded: bool = True
|
||||||
|
|
||||||
|
|
||||||
class LoginResponse(BaseModel):
|
class LoginResponse(BaseModel):
|
||||||
access_token: str
|
access_token: str
|
||||||
token_type: str = "bearer"
|
token_type: str = "bearer"
|
||||||
@@ -159,6 +215,9 @@ class LoginResponse(BaseModel):
|
|||||||
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||||
enabled_language_codes: list[str] = Field(default_factory=list)
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
default_language: str = "en"
|
default_language: str = "en"
|
||||||
|
profile_loaded: bool = True
|
||||||
|
roles_loaded: bool = True
|
||||||
|
groups_loaded: bool = True
|
||||||
|
|
||||||
|
|
||||||
class MeResponse(BaseModel):
|
class MeResponse(BaseModel):
|
||||||
@@ -174,3 +233,6 @@ class MeResponse(BaseModel):
|
|||||||
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||||
enabled_language_codes: list[str] = Field(default_factory=list)
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
default_language: str = "en"
|
default_language: str = "en"
|
||||||
|
profile_loaded: bool = True
|
||||||
|
roles_loaded: bool = True
|
||||||
|
groups_loaded: bool = True
|
||||||
|
|||||||
@@ -20,30 +20,13 @@ from govoplan_core.core.events import (
|
|||||||
)
|
)
|
||||||
from govoplan_core.core.runtime import get_registry
|
from govoplan_core.core.runtime import get_registry
|
||||||
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
||||||
|
from govoplan_core.security.redaction import redact_secret_values
|
||||||
|
|
||||||
|
|
||||||
AUDIT_MODULE_ID = "audit"
|
AUDIT_MODULE_ID = "audit"
|
||||||
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
|
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
|
||||||
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
|
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
|
||||||
|
|
||||||
SENSITIVE_DETAIL_KEYS = {
|
|
||||||
"password",
|
|
||||||
"smtp_password",
|
|
||||||
"imap_password",
|
|
||||||
"secret",
|
|
||||||
"api_key",
|
|
||||||
"token",
|
|
||||||
"access_token",
|
|
||||||
"refresh_token",
|
|
||||||
"authorization",
|
|
||||||
"cookie",
|
|
||||||
"credentials",
|
|
||||||
"credential",
|
|
||||||
"private_key",
|
|
||||||
"claim_token",
|
|
||||||
"build_token",
|
|
||||||
}
|
|
||||||
|
|
||||||
_ACTION_MODULE_PREFIXES = {
|
_ACTION_MODULE_PREFIXES = {
|
||||||
"api_key": "access",
|
"api_key": "access",
|
||||||
"configuration_change": "access",
|
"configuration_change": "access",
|
||||||
@@ -90,17 +73,7 @@ def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def _sanitize_details(value: Any) -> Any:
|
def _sanitize_details(value: Any) -> Any:
|
||||||
if isinstance(value, dict):
|
return redact_secret_values(value)
|
||||||
sanitized: dict[str, Any] = {}
|
|
||||||
for key, item in value.items():
|
|
||||||
if str(key).lower() in SENSITIVE_DETAIL_KEYS:
|
|
||||||
sanitized[key] = "<redacted>"
|
|
||||||
else:
|
|
||||||
sanitized[key] = _sanitize_details(item)
|
|
||||||
return sanitized
|
|
||||||
if isinstance(value, list):
|
|
||||||
return [_sanitize_details(item) for item in value]
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def audit_operation_context(
|
def audit_operation_context(
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ from celery import Celery
|
|||||||
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider
|
from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider
|
||||||
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||||
from govoplan_core.core.modules import ModuleContext
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.notifications import CAPABILITY_NOTIFICATIONS_DISPATCH, NotificationDispatchProvider
|
||||||
|
from govoplan_core.core.registry import PlatformRegistry
|
||||||
from govoplan_core.core.runtime import configure_runtime
|
from govoplan_core.core.runtime import configure_runtime
|
||||||
from govoplan_core.settings import settings
|
from govoplan_core.settings import settings
|
||||||
from govoplan_core.db.session import configure_database
|
from govoplan_core.db.session import configure_database
|
||||||
@@ -13,7 +15,7 @@ from govoplan_core.server.registry import available_module_manifests, build_plat
|
|||||||
configure_database(settings.database_url)
|
configure_database(settings.database_url)
|
||||||
|
|
||||||
celery = Celery(
|
celery = Celery(
|
||||||
"multimailer",
|
"govoplan",
|
||||||
broker=settings.redis_url,
|
broker=settings.redis_url,
|
||||||
backend=settings.redis_url,
|
backend=settings.redis_url,
|
||||||
)
|
)
|
||||||
@@ -21,8 +23,10 @@ celery = Celery(
|
|||||||
celery.conf.update(
|
celery.conf.update(
|
||||||
task_default_queue="default",
|
task_default_queue="default",
|
||||||
task_routes={
|
task_routes={
|
||||||
"multimailer.send_email": {"queue": "send_email"},
|
"govoplan.campaigns.send_email": {"queue": "send_email"},
|
||||||
"multimailer.append_sent": {"queue": "append_sent"},
|
"govoplan.campaigns.append_sent": {"queue": "append_sent"},
|
||||||
|
"govoplan.notifications.deliver": {"queue": "notifications"},
|
||||||
|
"govoplan.notifications.deliver_pending": {"queue": "notifications"},
|
||||||
},
|
},
|
||||||
worker_prefetch_multiplier=1,
|
worker_prefetch_multiplier=1,
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
@@ -30,12 +34,12 @@ celery.conf.update(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="multimailer.ping")
|
@celery.task(name="govoplan.ping")
|
||||||
def ping():
|
def ping():
|
||||||
return "pong"
|
return "pong"
|
||||||
|
|
||||||
|
|
||||||
def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
def _platform_registry() -> PlatformRegistry:
|
||||||
raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules)
|
raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules)
|
||||||
candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules)
|
candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules)
|
||||||
available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True)
|
available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True)
|
||||||
@@ -44,13 +48,26 @@ def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
|||||||
context = ModuleContext(registry=registry, settings=settings)
|
context = ModuleContext(registry=registry, settings=settings)
|
||||||
configure_runtime(context)
|
configure_runtime(context)
|
||||||
registry.configure_capability_context(context)
|
registry.configure_capability_context(context)
|
||||||
|
return registry
|
||||||
|
|
||||||
|
|
||||||
|
def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider:
|
||||||
|
registry = _platform_registry()
|
||||||
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_DELIVERY_TASKS)
|
capability = registry.require_capability(CAPABILITY_CAMPAIGNS_DELIVERY_TASKS)
|
||||||
if not isinstance(capability, CampaignDeliveryTaskProvider):
|
if not isinstance(capability, CampaignDeliveryTaskProvider):
|
||||||
raise RuntimeError("Campaign delivery task capability is invalid")
|
raise RuntimeError("Campaign delivery task capability is invalid")
|
||||||
return capability
|
return capability
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="multimailer.send_email", bind=True, max_retries=0)
|
def _notification_dispatch() -> NotificationDispatchProvider:
|
||||||
|
registry = _platform_registry()
|
||||||
|
capability = registry.require_capability(CAPABILITY_NOTIFICATIONS_DISPATCH)
|
||||||
|
if not isinstance(capability, NotificationDispatchProvider):
|
||||||
|
raise RuntimeError("Notification dispatch capability is invalid")
|
||||||
|
return capability
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="govoplan.campaigns.send_email", bind=True, max_retries=0)
|
||||||
def send_email(self, job_id: str):
|
def send_email(self, job_id: str):
|
||||||
"""Send one explicitly queued campaign job.
|
"""Send one explicitly queued campaign job.
|
||||||
|
|
||||||
@@ -65,7 +82,7 @@ def send_email(self, job_id: str):
|
|||||||
return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True))
|
return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True))
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="multimailer.append_sent", bind=True, max_retries=None)
|
@celery.task(name="govoplan.campaigns.append_sent", bind=True, max_retries=None)
|
||||||
def append_sent(self, job_id: str):
|
def append_sent(self, job_id: str):
|
||||||
"""Append the exact sent MIME to the configured IMAP Sent folder."""
|
"""Append the exact sent MIME to the configured IMAP Sent folder."""
|
||||||
|
|
||||||
@@ -78,3 +95,19 @@ def append_sent(self, job_id: str):
|
|||||||
if getattr(exc, "temporary", None) is True:
|
if getattr(exc, "temporary", None) is True:
|
||||||
raise self.retry(exc=exc, countdown=300)
|
raise self.retry(exc=exc, countdown=300)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="govoplan.notifications.deliver", bind=True, max_retries=0)
|
||||||
|
def deliver_notification(self, notification_id: str):
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
|
||||||
|
with get_database().SessionLocal() as session:
|
||||||
|
return dict(_notification_dispatch().deliver_notification(session, notification_id=notification_id))
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="govoplan.notifications.deliver_pending", bind=True, max_retries=0)
|
||||||
|
def deliver_pending_notifications(self, tenant_id: str | None = None, limit: int = 50):
|
||||||
|
from govoplan_core.db.session import get_database
|
||||||
|
|
||||||
|
with get_database().SessionLocal() as session:
|
||||||
|
return dict(_notification_dispatch().deliver_pending(session, tenant_id=tenant_id, limit=limit))
|
||||||
|
|||||||
@@ -1,9 +1,17 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from govoplan_core.db.bootstrap import bootstrap_dev_data
|
from govoplan_core.db.bootstrap import bootstrap_dev_data
|
||||||
from govoplan_core.db.migrations import migrate_database
|
from govoplan_core.db.migrations import (
|
||||||
|
ModuleMigrationTaskExecutionError,
|
||||||
|
POST_MIGRATION_TASK_PHASES,
|
||||||
|
PRE_MIGRATION_TASK_PHASES,
|
||||||
|
migrate_database,
|
||||||
|
run_registered_module_migration_tasks,
|
||||||
|
)
|
||||||
from govoplan_core.db.session import configure_database, get_database
|
from govoplan_core.db.session import configure_database, get_database
|
||||||
from govoplan_core.settings import settings
|
from govoplan_core.settings import settings
|
||||||
|
|
||||||
@@ -11,12 +19,50 @@ from govoplan_core.settings import settings
|
|||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
|
parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database")
|
||||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
|
parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate")
|
||||||
|
parser.add_argument(
|
||||||
|
"--migration-track",
|
||||||
|
default=settings.migration_track,
|
||||||
|
choices=("release", "dev"),
|
||||||
|
help="Alembic migration track to load: release uses squashed release shortcuts; dev uses detailed development chains.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.")
|
||||||
|
parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.")
|
||||||
|
parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.")
|
||||||
parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key")
|
parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key")
|
||||||
parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create")
|
parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
configure_database(args.database_url)
|
configure_database(args.database_url)
|
||||||
migration = migrate_database(database_url=args.database_url)
|
enabled_modules = tuple(args.enabled_module) if args.enabled_module else None
|
||||||
|
migration_order = tuple(args.migration_module) if args.migration_module else None
|
||||||
|
task_records: list[dict[str, object]] = []
|
||||||
|
try:
|
||||||
|
_run_migration_tasks(
|
||||||
|
task_records,
|
||||||
|
database_url=args.database_url,
|
||||||
|
enabled_modules=enabled_modules,
|
||||||
|
migration_order=migration_order,
|
||||||
|
phases=PRE_MIGRATION_TASK_PHASES,
|
||||||
|
)
|
||||||
|
migration = migrate_database(
|
||||||
|
database_url=args.database_url,
|
||||||
|
enabled_modules=enabled_modules,
|
||||||
|
migration_module_order=migration_order,
|
||||||
|
migration_track=args.migration_track,
|
||||||
|
)
|
||||||
|
_run_migration_tasks(
|
||||||
|
task_records,
|
||||||
|
database_url=args.database_url,
|
||||||
|
enabled_modules=enabled_modules,
|
||||||
|
migration_order=migration_order,
|
||||||
|
phases=POST_MIGRATION_TASK_PHASES,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if args.migration_task_record_output:
|
||||||
|
args.migration_task_record_output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.migration_task_record_output.write_text(json.dumps(task_records, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
if task_records:
|
||||||
|
print(f"Executed {len(task_records)} module migration task(s).")
|
||||||
if migration.reconciled_revision:
|
if migration.reconciled_revision:
|
||||||
print(f"Reconciled legacy database marker to {migration.reconciled_revision}.")
|
print(f"Reconciled legacy database marker to {migration.reconciled_revision}.")
|
||||||
print(f"Database schema upgraded to {migration.current_revision}.")
|
print(f"Database schema upgraded to {migration.current_revision}.")
|
||||||
@@ -33,5 +79,26 @@ def main() -> None:
|
|||||||
print("Development API key already exists or was not requested.")
|
print("Development API key already exists or was not requested.")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_migration_tasks(
|
||||||
|
target: list[dict[str, object]],
|
||||||
|
*,
|
||||||
|
database_url: str,
|
||||||
|
enabled_modules: tuple[str, ...] | None,
|
||||||
|
migration_order: tuple[str, ...] | None,
|
||||||
|
phases: tuple[str, ...],
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
records = run_registered_module_migration_tasks(
|
||||||
|
database_url=database_url,
|
||||||
|
enabled_modules=enabled_modules,
|
||||||
|
migration_module_order=migration_order,
|
||||||
|
phases=phases,
|
||||||
|
)
|
||||||
|
except ModuleMigrationTaskExecutionError as exc:
|
||||||
|
target.extend(exc.records)
|
||||||
|
raise
|
||||||
|
target.extend(records)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from govoplan_core.core.module_installer import (
|
from govoplan_core.core.module_installer import (
|
||||||
ModuleInstallerError,
|
ModuleInstallerError,
|
||||||
@@ -27,6 +26,13 @@ from govoplan_core.core.module_installer import (
|
|||||||
update_module_installer_request,
|
update_module_installer_request,
|
||||||
update_module_installer_daemon_status,
|
update_module_installer_daemon_status,
|
||||||
)
|
)
|
||||||
|
from govoplan_core.core.module_installer_notifications import (
|
||||||
|
build_runtime_notification_registry,
|
||||||
|
emit_module_installer_notification,
|
||||||
|
installer_notification_body,
|
||||||
|
installer_notification_priority,
|
||||||
|
installer_notification_subject,
|
||||||
|
)
|
||||||
from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics
|
from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics
|
||||||
from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog
|
from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog
|
||||||
from govoplan_core.core.module_management import (
|
from govoplan_core.core.module_management import (
|
||||||
@@ -39,7 +45,7 @@ from govoplan_core.server.registry import available_module_manifests
|
|||||||
from govoplan_core.settings import settings
|
from govoplan_core.settings import settings
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def _build_parser() -> argparse.ArgumentParser:
|
||||||
parser = argparse.ArgumentParser(description="Preflight, apply, or roll back a GovOPlaN module package install plan.")
|
parser = argparse.ArgumentParser(description="Preflight, apply, or roll back a GovOPlaN module package install plan.")
|
||||||
parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.")
|
parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.")
|
||||||
parser.add_argument("--runtime-dir", type=Path, help="Directory for installer locks and run snapshots.")
|
parser.add_argument("--runtime-dir", type=Path, help="Directory for installer locks and run snapshots.")
|
||||||
@@ -95,202 +101,295 @@ def main() -> int:
|
|||||||
parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.")
|
parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.")
|
||||||
parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.")
|
parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.")
|
||||||
parser.add_argument("--license-notes", help="Optional operator note for --issue-license.")
|
parser.add_argument("--license-notes", help="Optional operator note for --issue-license.")
|
||||||
args = parser.parse_args()
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = _build_parser().parse_args()
|
||||||
runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url)
|
runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url)
|
||||||
try:
|
try:
|
||||||
if args.issue_license:
|
return _dispatch_command(args=args, runtime_dir=runtime_dir)
|
||||||
if not args.license_id or not args.license_subject or not args.license_valid_until:
|
|
||||||
raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.")
|
|
||||||
if not args.license_signing_key_id or not args.license_signing_private_key:
|
|
||||||
raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.")
|
|
||||||
if not [item for item in args.license_feature if item.strip()]:
|
|
||||||
raise ModuleInstallerError("--issue-license requires at least one --license-feature.")
|
|
||||||
try:
|
|
||||||
path = issue_module_license(
|
|
||||||
path=args.issue_license,
|
|
||||||
license_id=args.license_id,
|
|
||||||
subject=args.license_subject,
|
|
||||||
features=args.license_feature,
|
|
||||||
valid_from=args.license_valid_from,
|
|
||||||
valid_until=args.license_valid_until,
|
|
||||||
signing_key_id=args.license_signing_key_id,
|
|
||||||
signing_private_key_path=args.license_signing_private_key,
|
|
||||||
issuer=args.license_issuer,
|
|
||||||
notes=args.license_notes,
|
|
||||||
)
|
|
||||||
except (OSError, ValueError) as exc:
|
|
||||||
raise ModuleInstallerError(str(exc)) from exc
|
|
||||||
_print_result(
|
|
||||||
{
|
|
||||||
"issued": True,
|
|
||||||
"path": str(path),
|
|
||||||
"license_id": args.license_id,
|
|
||||||
"subject": args.license_subject,
|
|
||||||
"features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())),
|
|
||||||
"valid_until": args.license_valid_until,
|
|
||||||
"key_id": args.license_signing_key_id,
|
|
||||||
},
|
|
||||||
output_format=args.format,
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
if args.validate_license is not None:
|
|
||||||
path = Path(args.validate_license).expanduser() if args.validate_license else None
|
|
||||||
result = module_license_diagnostics(
|
|
||||||
path,
|
|
||||||
require_trusted=args.require_trusted_license,
|
|
||||||
trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"),
|
|
||||||
required_features=args.license_required_feature,
|
|
||||||
)
|
|
||||||
_print_result(result, output_format=args.format)
|
|
||||||
return 0 if result.get("valid") and not result.get("missing_features") else 1
|
|
||||||
if args.sign_package_catalog:
|
|
||||||
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
|
||||||
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
|
||||||
path = sign_module_package_catalog(
|
|
||||||
path=args.sign_package_catalog,
|
|
||||||
key_id=args.catalog_signing_key_id,
|
|
||||||
private_key_path=args.catalog_signing_private_key,
|
|
||||||
output_path=args.catalog_output,
|
|
||||||
)
|
|
||||||
_print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.validate_package_catalog is not None:
|
|
||||||
path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None
|
|
||||||
result = validate_module_package_catalog(
|
|
||||||
path,
|
|
||||||
require_trusted=args.require_signed_catalog,
|
|
||||||
approved_channels=tuple(args.approved_catalog_channel),
|
|
||||||
trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"),
|
|
||||||
)
|
|
||||||
_print_result(result, output_format=args.format)
|
|
||||||
return 0 if result.get("valid") else 1
|
|
||||||
if args.daemon_status:
|
|
||||||
_print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.daemon or args.daemon_once:
|
|
||||||
return _run_daemon(args=args, runtime_dir=runtime_dir)
|
|
||||||
if args.list_requests:
|
|
||||||
_print_result({
|
|
||||||
"requests": list(list_module_installer_requests(runtime_dir=runtime_dir)),
|
|
||||||
"daemon": module_installer_daemon_status(runtime_dir=runtime_dir),
|
|
||||||
}, output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.show_request:
|
|
||||||
_print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.cancel_request:
|
|
||||||
_print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.retry_request:
|
|
||||||
_print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.enqueue_supervised:
|
|
||||||
request = queue_module_installer_request(
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
requested_by="cli",
|
|
||||||
options=_request_options_from_args(args),
|
|
||||||
)
|
|
||||||
_print_result(request, output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.list_runs:
|
|
||||||
_print_result({
|
|
||||||
"runs": list(list_module_installer_runs(runtime_dir=runtime_dir)),
|
|
||||||
"lock": module_installer_lock_status(runtime_dir=runtime_dir),
|
|
||||||
}, output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.show_run:
|
|
||||||
_print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.lock_status:
|
|
||||||
_print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format)
|
|
||||||
return 0
|
|
||||||
if args.rollback:
|
|
||||||
result = rollback_module_install_run(
|
|
||||||
run_id=args.rollback,
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
npm_bin=args.npm_bin,
|
|
||||||
build_webui=args.build_webui,
|
|
||||||
database_restore_command=args.database_restore_command,
|
|
||||||
database_url=str(args.database_url),
|
|
||||||
)
|
|
||||||
_print_result(result.as_dict(), output_format=args.format)
|
|
||||||
return 0 if result.return_code == 0 else 1
|
|
||||||
|
|
||||||
configure_database(str(args.database_url))
|
|
||||||
available = available_module_manifests(ignore_load_errors=True)
|
|
||||||
with get_database().session() as session:
|
|
||||||
configured = configured_enabled_modules(settings.enabled_modules)
|
|
||||||
desired = saved_desired_enabled_modules(session, configured)
|
|
||||||
plan = saved_module_install_plan(session)
|
|
||||||
if args.supervise:
|
|
||||||
result = supervise_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available,
|
|
||||||
current_enabled=desired,
|
|
||||||
desired_enabled=desired,
|
|
||||||
database_url=str(args.database_url),
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
webui_root=args.webui_root,
|
|
||||||
npm_bin=args.npm_bin,
|
|
||||||
build_webui=args.build_webui,
|
|
||||||
migrate_database=args.migrate,
|
|
||||||
database_backup_command=args.database_backup_command,
|
|
||||||
database_restore_command=args.database_restore_command,
|
|
||||||
database_restore_check_command=args.database_restore_check_command,
|
|
||||||
activate_installed_modules=not args.no_activate_installed_modules,
|
|
||||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
|
||||||
restart_command=args.restart_command,
|
|
||||||
health_url=args.health_url,
|
|
||||||
health_timeout_seconds=args.health_timeout_seconds,
|
|
||||||
health_interval_seconds=args.health_interval_seconds,
|
|
||||||
)
|
|
||||||
_print_result(result.as_dict(), output_format=args.format)
|
|
||||||
return 0 if result.return_code == 0 else 1
|
|
||||||
if args.apply or args.dry_run:
|
|
||||||
result = run_module_install_plan(
|
|
||||||
session=session,
|
|
||||||
plan=plan,
|
|
||||||
available=available,
|
|
||||||
current_enabled=desired,
|
|
||||||
desired_enabled=desired,
|
|
||||||
database_url=str(args.database_url),
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
webui_root=args.webui_root,
|
|
||||||
npm_bin=args.npm_bin,
|
|
||||||
build_webui=args.build_webui,
|
|
||||||
migrate_database=args.migrate,
|
|
||||||
database_backup_command=args.database_backup_command,
|
|
||||||
database_restore_command=args.database_restore_command,
|
|
||||||
database_restore_check_command=args.database_restore_check_command,
|
|
||||||
activate_installed_modules=not args.no_activate_installed_modules,
|
|
||||||
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
|
||||||
dry_run=args.dry_run,
|
|
||||||
)
|
|
||||||
_print_result(result.as_dict(), output_format=args.format)
|
|
||||||
return 0 if result.return_code == 0 else 1
|
|
||||||
|
|
||||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
|
||||||
|
|
||||||
maintenance = saved_maintenance_mode(session)
|
|
||||||
preflight = module_install_preflight(
|
|
||||||
plan=plan,
|
|
||||||
available=available,
|
|
||||||
current_enabled=desired,
|
|
||||||
desired_enabled=desired,
|
|
||||||
maintenance_mode=maintenance.enabled,
|
|
||||||
session=session,
|
|
||||||
webui_root=args.webui_root,
|
|
||||||
runtime_dir=runtime_dir,
|
|
||||||
)
|
|
||||||
_print_preflight(preflight.as_dict(), output_format=args.format)
|
|
||||||
return 0 if preflight.allowed else 1
|
|
||||||
except ModuleInstallerError as exc:
|
except ModuleInstallerError as exc:
|
||||||
print(f"error: {exc}", file=sys.stderr)
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def _dispatch_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||||
|
for handler in (
|
||||||
|
_handle_license_command,
|
||||||
|
_handle_catalog_command,
|
||||||
|
_handle_runtime_command,
|
||||||
|
_handle_queue_command,
|
||||||
|
_handle_run_history_command,
|
||||||
|
):
|
||||||
|
result = handler(args=args, runtime_dir=runtime_dir)
|
||||||
|
if result is not None:
|
||||||
|
return result
|
||||||
|
return _handle_install_command(args=args, runtime_dir=runtime_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_license_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||||
|
del runtime_dir
|
||||||
|
if args.issue_license:
|
||||||
|
return _issue_license_command(args)
|
||||||
|
if args.validate_license is not None:
|
||||||
|
return _validate_license_command(args)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _issue_license_command(args: argparse.Namespace) -> int:
|
||||||
|
if not args.license_id or not args.license_subject or not args.license_valid_until:
|
||||||
|
raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.")
|
||||||
|
if not args.license_signing_key_id or not args.license_signing_private_key:
|
||||||
|
raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.")
|
||||||
|
if not [item for item in args.license_feature if item.strip()]:
|
||||||
|
raise ModuleInstallerError("--issue-license requires at least one --license-feature.")
|
||||||
|
try:
|
||||||
|
path = issue_module_license(
|
||||||
|
path=args.issue_license,
|
||||||
|
license_id=args.license_id,
|
||||||
|
subject=args.license_subject,
|
||||||
|
features=args.license_feature,
|
||||||
|
valid_from=args.license_valid_from,
|
||||||
|
valid_until=args.license_valid_until,
|
||||||
|
signing_key_id=args.license_signing_key_id,
|
||||||
|
signing_private_key_path=args.license_signing_private_key,
|
||||||
|
issuer=args.license_issuer,
|
||||||
|
notes=args.license_notes,
|
||||||
|
)
|
||||||
|
except (OSError, ValueError) as exc:
|
||||||
|
raise ModuleInstallerError(str(exc)) from exc
|
||||||
|
_print_result(
|
||||||
|
{
|
||||||
|
"issued": True,
|
||||||
|
"path": str(path),
|
||||||
|
"license_id": args.license_id,
|
||||||
|
"subject": args.license_subject,
|
||||||
|
"features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())),
|
||||||
|
"valid_until": args.license_valid_until,
|
||||||
|
"key_id": args.license_signing_key_id,
|
||||||
|
},
|
||||||
|
output_format=args.format,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_license_command(args: argparse.Namespace) -> int:
|
||||||
|
path = Path(args.validate_license).expanduser() if args.validate_license else None
|
||||||
|
result = module_license_diagnostics(
|
||||||
|
path,
|
||||||
|
require_trusted=args.require_trusted_license,
|
||||||
|
trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"),
|
||||||
|
required_features=args.license_required_feature,
|
||||||
|
)
|
||||||
|
_print_result(result, output_format=args.format)
|
||||||
|
return 0 if result.get("valid") and not result.get("missing_features") else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_catalog_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||||
|
del runtime_dir
|
||||||
|
if args.sign_package_catalog:
|
||||||
|
return _sign_package_catalog_command(args)
|
||||||
|
if args.validate_package_catalog is not None:
|
||||||
|
return _validate_package_catalog_command(args)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _sign_package_catalog_command(args: argparse.Namespace) -> int:
|
||||||
|
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
||||||
|
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
||||||
|
path = sign_module_package_catalog(
|
||||||
|
path=args.sign_package_catalog,
|
||||||
|
key_id=args.catalog_signing_key_id,
|
||||||
|
private_key_path=args.catalog_signing_private_key,
|
||||||
|
output_path=args.catalog_output,
|
||||||
|
)
|
||||||
|
_print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_package_catalog_command(args: argparse.Namespace) -> int:
|
||||||
|
path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None
|
||||||
|
result = validate_module_package_catalog(
|
||||||
|
path,
|
||||||
|
require_trusted=args.require_signed_catalog,
|
||||||
|
approved_channels=tuple(args.approved_catalog_channel),
|
||||||
|
trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"),
|
||||||
|
)
|
||||||
|
_print_result(result, output_format=args.format)
|
||||||
|
return 0 if result.get("valid") else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_runtime_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||||
|
if args.daemon_status:
|
||||||
|
_print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.daemon or args.daemon_once:
|
||||||
|
return _run_daemon(args=args, runtime_dir=runtime_dir)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_queue_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||||
|
if args.list_requests:
|
||||||
|
_print_result({
|
||||||
|
"requests": list(list_module_installer_requests(runtime_dir=runtime_dir)),
|
||||||
|
"daemon": module_installer_daemon_status(runtime_dir=runtime_dir),
|
||||||
|
}, output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.show_request:
|
||||||
|
_print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.cancel_request:
|
||||||
|
_print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.retry_request:
|
||||||
|
_print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.enqueue_supervised:
|
||||||
|
request = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="cli", options=_request_options_from_args(args))
|
||||||
|
_print_result(request, output_format=args.format)
|
||||||
|
return 0
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_run_history_command(*, args: argparse.Namespace, runtime_dir: Path) -> int | None:
|
||||||
|
if args.list_runs:
|
||||||
|
_print_result({
|
||||||
|
"runs": list(list_module_installer_runs(runtime_dir=runtime_dir)),
|
||||||
|
"lock": module_installer_lock_status(runtime_dir=runtime_dir),
|
||||||
|
}, output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.show_run:
|
||||||
|
_print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.lock_status:
|
||||||
|
_print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format)
|
||||||
|
return 0
|
||||||
|
if args.rollback:
|
||||||
|
return _rollback_command(args=args, runtime_dir=runtime_dir)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _rollback_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||||
|
result = rollback_module_install_run(
|
||||||
|
run_id=args.rollback,
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
npm_bin=args.npm_bin,
|
||||||
|
build_webui=args.build_webui,
|
||||||
|
database_restore_command=args.database_restore_command,
|
||||||
|
database_url=str(args.database_url),
|
||||||
|
)
|
||||||
|
_print_result(result.as_dict(), output_format=args.format)
|
||||||
|
return 0 if result.return_code == 0 else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_install_command(*, args: argparse.Namespace, runtime_dir: Path) -> int:
|
||||||
|
configure_database(str(args.database_url))
|
||||||
|
available = available_module_manifests(ignore_load_errors=True)
|
||||||
|
with get_database().session() as session:
|
||||||
|
configured = configured_enabled_modules(settings.enabled_modules)
|
||||||
|
desired = saved_desired_enabled_modules(session, configured)
|
||||||
|
plan = saved_module_install_plan(session)
|
||||||
|
if args.supervise:
|
||||||
|
return _supervise_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||||
|
if args.apply or args.dry_run:
|
||||||
|
return _apply_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||||
|
return _preflight_install_command(args=args, runtime_dir=runtime_dir, session=session, available=available, desired=desired, plan=plan)
|
||||||
|
|
||||||
|
|
||||||
|
def _supervise_install_command(
|
||||||
|
*,
|
||||||
|
args: argparse.Namespace,
|
||||||
|
runtime_dir: Path,
|
||||||
|
session: object,
|
||||||
|
available: object,
|
||||||
|
desired: object,
|
||||||
|
plan: object,
|
||||||
|
) -> int:
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available,
|
||||||
|
current_enabled=desired,
|
||||||
|
desired_enabled=desired,
|
||||||
|
database_url=str(args.database_url),
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
webui_root=args.webui_root,
|
||||||
|
npm_bin=args.npm_bin,
|
||||||
|
build_webui=args.build_webui,
|
||||||
|
migrate_database=args.migrate,
|
||||||
|
database_backup_command=args.database_backup_command,
|
||||||
|
database_restore_command=args.database_restore_command,
|
||||||
|
database_restore_check_command=args.database_restore_check_command,
|
||||||
|
activate_installed_modules=not args.no_activate_installed_modules,
|
||||||
|
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||||
|
restart_command=args.restart_command,
|
||||||
|
health_url=args.health_url,
|
||||||
|
health_timeout_seconds=args.health_timeout_seconds,
|
||||||
|
health_interval_seconds=args.health_interval_seconds,
|
||||||
|
)
|
||||||
|
_print_result(result.as_dict(), output_format=args.format)
|
||||||
|
return 0 if result.return_code == 0 else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_install_command(
|
||||||
|
*,
|
||||||
|
args: argparse.Namespace,
|
||||||
|
runtime_dir: Path,
|
||||||
|
session: object,
|
||||||
|
available: object,
|
||||||
|
desired: object,
|
||||||
|
plan: object,
|
||||||
|
) -> int:
|
||||||
|
result = run_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available,
|
||||||
|
current_enabled=desired,
|
||||||
|
desired_enabled=desired,
|
||||||
|
database_url=str(args.database_url),
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
webui_root=args.webui_root,
|
||||||
|
npm_bin=args.npm_bin,
|
||||||
|
build_webui=args.build_webui,
|
||||||
|
migrate_database=args.migrate,
|
||||||
|
database_backup_command=args.database_backup_command,
|
||||||
|
database_restore_command=args.database_restore_command,
|
||||||
|
database_restore_check_command=args.database_restore_check_command,
|
||||||
|
activate_installed_modules=not args.no_activate_installed_modules,
|
||||||
|
remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
)
|
||||||
|
_print_result(result.as_dict(), output_format=args.format)
|
||||||
|
return 0 if result.return_code == 0 else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _preflight_install_command(
|
||||||
|
*,
|
||||||
|
args: argparse.Namespace,
|
||||||
|
runtime_dir: Path,
|
||||||
|
session: object,
|
||||||
|
available: object,
|
||||||
|
desired: object,
|
||||||
|
plan: object,
|
||||||
|
) -> int:
|
||||||
|
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||||
|
|
||||||
|
maintenance = saved_maintenance_mode(session)
|
||||||
|
preflight = module_install_preflight(
|
||||||
|
plan=plan,
|
||||||
|
available=available,
|
||||||
|
current_enabled=desired,
|
||||||
|
desired_enabled=desired,
|
||||||
|
maintenance_mode=maintenance.enabled,
|
||||||
|
session=session,
|
||||||
|
webui_root=args.webui_root,
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
)
|
||||||
|
_print_preflight(preflight.as_dict(), output_format=args.format)
|
||||||
|
return 0 if preflight.allowed else 1
|
||||||
|
|
||||||
|
|
||||||
def _default_webui_root() -> Path:
|
def _default_webui_root() -> Path:
|
||||||
return Path(__file__).resolve().parents[3] / "webui"
|
return Path(__file__).resolve().parents[3] / "webui"
|
||||||
|
|
||||||
@@ -355,6 +454,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
|||||||
request_id = str(request["request_id"])
|
request_id = str(request["request_id"])
|
||||||
options = request.get("options") if isinstance(request.get("options"), dict) else {}
|
options = request.get("options") if isinstance(request.get("options"), dict) else {}
|
||||||
configure_database(str(args.database_url))
|
configure_database(str(args.database_url))
|
||||||
|
_emit_installer_daemon_notification(request, event_kind="module_installer.request.running")
|
||||||
try:
|
try:
|
||||||
available = available_module_manifests(ignore_load_errors=True)
|
available = available_module_manifests(ignore_load_errors=True)
|
||||||
with get_database().session() as session:
|
with get_database().session() as session:
|
||||||
@@ -384,7 +484,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
|||||||
health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds),
|
health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds),
|
||||||
request_context=_request_context(request),
|
request_context=_request_context(request),
|
||||||
)
|
)
|
||||||
update_module_installer_request(
|
updated_request = update_module_installer_request(
|
||||||
runtime_dir=runtime_dir,
|
runtime_dir=runtime_dir,
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
patch={
|
patch={
|
||||||
@@ -393,8 +493,12 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
|||||||
"result": result.as_dict(),
|
"result": result.as_dict(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
_emit_installer_daemon_notification(
|
||||||
|
updated_request,
|
||||||
|
event_kind="module_installer.request.completed" if result.return_code == 0 else "module_installer.request.failed",
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
update_module_installer_request(
|
updated_request = update_module_installer_request(
|
||||||
runtime_dir=runtime_dir,
|
runtime_dir=runtime_dir,
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
patch={
|
patch={
|
||||||
@@ -403,6 +507,34 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
|||||||
"error": str(exc),
|
"error": str(exc),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
_emit_installer_daemon_notification(updated_request, event_kind="module_installer.request.failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_installer_daemon_notification(request: dict[str, object], *, event_kind: str) -> None:
|
||||||
|
tenant_id = str(request.get("tenant_id") or "")
|
||||||
|
if not tenant_id:
|
||||||
|
return
|
||||||
|
registry = build_runtime_notification_registry(settings)
|
||||||
|
if registry is None:
|
||||||
|
return
|
||||||
|
status_value = str(request.get("status") or event_kind.rsplit(".", 1)[-1])
|
||||||
|
try:
|
||||||
|
with get_database().SessionLocal() as session:
|
||||||
|
emitted = emit_module_installer_notification(
|
||||||
|
session=session,
|
||||||
|
registry=registry,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
request=request,
|
||||||
|
event_kind=event_kind,
|
||||||
|
subject=installer_notification_subject(event_kind, request),
|
||||||
|
body_text=installer_notification_body(event_kind, request),
|
||||||
|
recipient_id=str(request.get("requested_by") or "") or None,
|
||||||
|
priority=installer_notification_priority(status_value),
|
||||||
|
)
|
||||||
|
if emitted:
|
||||||
|
session.commit()
|
||||||
|
except Exception: # noqa: BLE001 - notification bridge must not block installer work.
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]:
|
def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]:
|
||||||
@@ -427,6 +559,7 @@ def _request_context(request: dict[str, object]) -> dict[str, object]:
|
|||||||
context: dict[str, object] = {
|
context: dict[str, object] = {
|
||||||
"request_id": request.get("request_id"),
|
"request_id": request.get("request_id"),
|
||||||
"requested_by": request.get("requested_by"),
|
"requested_by": request.get("requested_by"),
|
||||||
|
"tenant_id": request.get("tenant_id"),
|
||||||
}
|
}
|
||||||
if request.get("retry_of"):
|
if request.get("retry_of"):
|
||||||
context["retry_of"] = request["retry_of"]
|
context["retry_of"] = request["retry_of"]
|
||||||
|
|||||||
@@ -85,6 +85,9 @@ AccessProvenanceKind = Literal[
|
|||||||
"identity",
|
"identity",
|
||||||
"account",
|
"account",
|
||||||
"tenant_membership",
|
"tenant_membership",
|
||||||
|
"resource",
|
||||||
|
"owner",
|
||||||
|
"share",
|
||||||
"organization_unit",
|
"organization_unit",
|
||||||
"function",
|
"function",
|
||||||
"group",
|
"group",
|
||||||
@@ -496,6 +499,20 @@ class ResourceAccessService(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class ResourceAccessExplanationProvider(Protocol):
|
||||||
|
def explain_resource_provenance(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
principal: PrincipalRef,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
action: str,
|
||||||
|
) -> Sequence[AccessDecisionProvenance]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
class AccessExplanationService(Protocol):
|
class AccessExplanationService(Protocol):
|
||||||
def explain_scope_provenance(
|
def explain_scope_provenance(
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from govoplan_core.admin.settings import get_system_settings
|
|||||||
from govoplan_core.core.change_sequence import record_change
|
from govoplan_core.core.change_sequence import record_change
|
||||||
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
|
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
|
||||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||||
|
from govoplan_core.security.redaction import redact_secret_values
|
||||||
from govoplan_core.security.time import utc_now
|
from govoplan_core.security.time import utc_now
|
||||||
|
|
||||||
|
|
||||||
@@ -377,17 +378,7 @@ def _sanitize_value(key: str, value: object) -> object:
|
|||||||
|
|
||||||
|
|
||||||
def _redact_secrets(value: object) -> object:
|
def _redact_secrets(value: object) -> object:
|
||||||
if isinstance(value, dict):
|
return redact_secret_values(value)
|
||||||
result: dict[str, object] = {}
|
|
||||||
for key, item in value.items():
|
|
||||||
if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
|
|
||||||
result[str(key)] = "<redacted>"
|
|
||||||
else:
|
|
||||||
result[str(key)] = _redact_secrets(item)
|
|
||||||
return result
|
|
||||||
if isinstance(value, list):
|
|
||||||
return [_redact_secrets(item) for item in value]
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ from pathlib import Path
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from typing import Any, Literal, Protocol, runtime_checkable
|
from typing import Any, Literal, Protocol, runtime_checkable
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
from govoplan_core.core.module_package_catalog import (
|
from govoplan_core.core.module_package_catalog import (
|
||||||
_canonical_catalog_bytes,
|
_canonical_catalog_bytes,
|
||||||
@@ -23,6 +21,7 @@ from govoplan_core.core.module_package_catalog import (
|
|||||||
_load_private_key,
|
_load_private_key,
|
||||||
_parse_trusted_keys,
|
_parse_trusted_keys,
|
||||||
)
|
)
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http_text
|
||||||
|
|
||||||
|
|
||||||
CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider"
|
CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider"
|
||||||
@@ -31,6 +30,20 @@ DiagnosticSeverity = Literal["blocker", "warning", "info"]
|
|||||||
PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"]
|
PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _ConfigurationCatalogValidationState:
|
||||||
|
packages: tuple[dict[str, object], ...]
|
||||||
|
channel: str | None
|
||||||
|
sequence: int | None
|
||||||
|
generated_at: str | None
|
||||||
|
not_before: str | None
|
||||||
|
expires_at: str | None
|
||||||
|
signature_state: dict[str, object]
|
||||||
|
freshness: dict[str, object]
|
||||||
|
replay: dict[str, object]
|
||||||
|
read_state: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class ConfigurationModuleRequirement:
|
class ConfigurationModuleRequirement:
|
||||||
module_id: str
|
module_id: str
|
||||||
@@ -461,55 +474,148 @@ def validate_configuration_package_catalog(
|
|||||||
configured = source is not None and _catalog_source_exists(source)
|
configured = source is not None and _catalog_source_exists(source)
|
||||||
if source is not None and not _catalog_source_exists(source):
|
if source is not None and not _catalog_source_exists(source):
|
||||||
return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}")
|
return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}")
|
||||||
read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() else None}
|
read_state = _configuration_catalog_default_read_state()
|
||||||
try:
|
try:
|
||||||
payload, read_state = _read_catalog_payload_with_metadata(source)
|
|
||||||
packages = _normalize_catalog_packages(payload)
|
|
||||||
channel = _catalog_channel(payload)
|
|
||||||
sequence = _catalog_sequence(payload)
|
|
||||||
generated_at = _catalog_optional_text(payload, "generated_at")
|
|
||||||
not_before = _catalog_optional_text(payload, "not_before")
|
|
||||||
expires_at = _catalog_optional_text(payload, "expires_at")
|
|
||||||
effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted
|
effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted
|
||||||
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
||||||
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
||||||
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
state = _configuration_catalog_validation_state(source, trusted_keys=effective_trusted_keys)
|
||||||
freshness = _catalog_freshness_state(payload)
|
read_state = state.read_state
|
||||||
replay = _catalog_replay_state(channel=channel, sequence=sequence)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc))
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc))
|
||||||
if signature_state.get("fatal"):
|
policy_error = _configuration_catalog_policy_error(
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"]))
|
source,
|
||||||
if effective_approved_channels and channel not in effective_approved_channels:
|
configured=configured,
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=f"Configuration package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.")
|
state=state,
|
||||||
if effective_require_trusted and not signature_state["trusted"]:
|
require_trusted=effective_require_trusted,
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"] or "Configuration package catalog must be signed by a trusted key."))
|
approved_channels=effective_approved_channels,
|
||||||
if not freshness["valid"]:
|
)
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(freshness["error"]))
|
if policy_error is not None:
|
||||||
if not replay["valid"]:
|
return policy_error
|
||||||
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(replay["error"]))
|
|
||||||
warnings = [str(item) for item in freshness.get("warnings", ()) if item]
|
|
||||||
warnings.extend(str(item) for item in replay.get("warnings", ()) if item)
|
|
||||||
if not signature_state["signed"]:
|
|
||||||
warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
|
||||||
elif not signature_state["trusted"]:
|
|
||||||
warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
|
||||||
return _validation_result(
|
return _validation_result(
|
||||||
source,
|
source,
|
||||||
valid=True,
|
valid=True,
|
||||||
configured=configured,
|
configured=configured,
|
||||||
read_state=read_state,
|
read_state=state.read_state,
|
||||||
packages=packages,
|
packages=state.packages,
|
||||||
|
channel=state.channel,
|
||||||
|
sequence=state.sequence,
|
||||||
|
generated_at=state.generated_at,
|
||||||
|
not_before=state.not_before,
|
||||||
|
expires_at=state.expires_at,
|
||||||
|
signature_state=state.signature_state,
|
||||||
|
warnings=_configuration_catalog_warnings(state),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_catalog_validation_state(
|
||||||
|
source: Path | str | None,
|
||||||
|
*,
|
||||||
|
trusted_keys: dict[str, str],
|
||||||
|
) -> _ConfigurationCatalogValidationState:
|
||||||
|
payload, read_state = _read_catalog_payload_with_metadata(source)
|
||||||
|
channel = _catalog_channel(payload)
|
||||||
|
sequence = _catalog_sequence(payload)
|
||||||
|
return _ConfigurationCatalogValidationState(
|
||||||
|
packages=_normalize_catalog_packages(payload),
|
||||||
channel=channel,
|
channel=channel,
|
||||||
sequence=sequence,
|
sequence=sequence,
|
||||||
generated_at=generated_at,
|
generated_at=_catalog_optional_text(payload, "generated_at"),
|
||||||
not_before=not_before,
|
not_before=_catalog_optional_text(payload, "not_before"),
|
||||||
expires_at=expires_at,
|
expires_at=_catalog_optional_text(payload, "expires_at"),
|
||||||
signature_state=signature_state,
|
signature_state=_catalog_signature_state(payload, trusted_keys=trusted_keys),
|
||||||
warnings=warnings,
|
freshness=_catalog_freshness_state(payload),
|
||||||
|
replay=_catalog_replay_state(channel=channel, sequence=sequence),
|
||||||
|
read_state=read_state,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_catalog_policy_error(
|
||||||
|
source: Path | str | None,
|
||||||
|
*,
|
||||||
|
configured: bool,
|
||||||
|
state: _ConfigurationCatalogValidationState,
|
||||||
|
require_trusted: bool,
|
||||||
|
approved_channels: tuple[str, ...],
|
||||||
|
) -> dict[str, object] | None:
|
||||||
|
if state.signature_state.get("fatal"):
|
||||||
|
return _configuration_catalog_invalid_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
state=state,
|
||||||
|
error=str(state.signature_state["error"]),
|
||||||
|
)
|
||||||
|
if approved_channels and state.channel not in approved_channels:
|
||||||
|
return _configuration_catalog_invalid_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
state=state,
|
||||||
|
error=(
|
||||||
|
f"Configuration package catalog channel {state.channel!r} is not approved. "
|
||||||
|
f"Approved channels: {', '.join(approved_channels)}."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if require_trusted and not state.signature_state["trusted"]:
|
||||||
|
return _configuration_catalog_invalid_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
state=state,
|
||||||
|
error=str(state.signature_state["error"] or "Configuration package catalog must be signed by a trusted key."),
|
||||||
|
)
|
||||||
|
if not state.freshness["valid"]:
|
||||||
|
return _configuration_catalog_invalid_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
state=state,
|
||||||
|
error=str(state.freshness["error"]),
|
||||||
|
)
|
||||||
|
if not state.replay["valid"]:
|
||||||
|
return _configuration_catalog_invalid_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
state=state,
|
||||||
|
error=str(state.replay["error"]),
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_catalog_invalid_result(
|
||||||
|
source: Path | str | None,
|
||||||
|
*,
|
||||||
|
configured: bool,
|
||||||
|
state: _ConfigurationCatalogValidationState,
|
||||||
|
error: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return _validation_result(
|
||||||
|
source,
|
||||||
|
configured=configured,
|
||||||
|
read_state=state.read_state,
|
||||||
|
packages=(),
|
||||||
|
channel=state.channel,
|
||||||
|
sequence=state.sequence,
|
||||||
|
generated_at=state.generated_at,
|
||||||
|
not_before=state.not_before,
|
||||||
|
expires_at=state.expires_at,
|
||||||
|
signature_state=state.signature_state,
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_catalog_warnings(state: _ConfigurationCatalogValidationState) -> list[str]:
|
||||||
|
warnings = [str(item) for item in state.freshness.get("warnings", ()) if item]
|
||||||
|
warnings.extend(str(item) for item in state.replay.get("warnings", ()) if item)
|
||||||
|
if not state.signature_state["signed"]:
|
||||||
|
warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
||||||
|
elif not state.signature_state["trusted"]:
|
||||||
|
warnings.append(str(state.signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
||||||
|
return warnings
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_catalog_default_read_state() -> dict[str, object]:
|
||||||
|
cache_path = _configured_catalog_cache_path()
|
||||||
|
return {"cache_used": False, "cache_path": str(cache_path) if cache_path else None}
|
||||||
|
|
||||||
|
|
||||||
def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path:
|
def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path:
|
||||||
payload = _read_catalog_payload(path)
|
payload = _read_catalog_payload(path)
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
@@ -686,17 +792,14 @@ def _configured_trusted_keys_cache_path() -> Path | None:
|
|||||||
|
|
||||||
|
|
||||||
def _read_trusted_keys_url(url: str) -> str:
|
def _read_trusted_keys_url(url: str) -> str:
|
||||||
if not _is_http_url(url):
|
|
||||||
raise ValueError("Trusted configuration catalog key URL must use http:// or https://.")
|
|
||||||
cache_path = _configured_trusted_keys_cache_path()
|
cache_path = _configured_trusted_keys_cache_path()
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=15) as response:
|
body = fetch_http_text(url, timeout=15, label="Trusted configuration catalog key URL")
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return body
|
return body
|
||||||
except (OSError, urllib.error.URLError):
|
except OSError:
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
return cache_path.read_text(encoding="utf-8")
|
return cache_path.read_text(encoding="utf-8")
|
||||||
raise
|
raise
|
||||||
@@ -714,13 +817,12 @@ def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[obje
|
|||||||
return {"packages": []}, metadata
|
return {"packages": []}, metadata
|
||||||
if isinstance(source, str) and _is_http_url(source):
|
if isinstance(source, str) and _is_http_url(source):
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(source, timeout=15) as response:
|
body = fetch_http_text(source, timeout=15, label="Configuration package catalog URL")
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return json.loads(body), metadata
|
return json.loads(body), metadata
|
||||||
except (OSError, urllib.error.URLError):
|
except OSError:
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
metadata["cache_used"] = True
|
metadata["cache_used"] = True
|
||||||
return json.loads(cache_path.read_text(encoding="utf-8")), metadata
|
return json.loads(cache_path.read_text(encoding="utf-8")), metadata
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from collections.abc import Mapping
|
|||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
from govoplan_core.security.permissions import scopes_grant
|
from govoplan_core.security.permissions import scopes_grant
|
||||||
|
from govoplan_core.security.redaction import contains_plain_secret
|
||||||
|
|
||||||
|
|
||||||
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
|
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
|
||||||
@@ -96,6 +97,16 @@ class ConfigurationChangeSafetyPlan:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _ConfigurationChangeSafetyState:
|
||||||
|
field: ConfigurationFieldSafety
|
||||||
|
missing_scopes: tuple[str, ...]
|
||||||
|
blockers: tuple[str, ...]
|
||||||
|
warnings: tuple[str, ...]
|
||||||
|
approval_required: bool
|
||||||
|
approval_satisfied: bool
|
||||||
|
|
||||||
|
|
||||||
_CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
_CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||||
ConfigurationFieldSafety(
|
ConfigurationFieldSafety(
|
||||||
key="module_management.desired_enabled",
|
key="module_management.desired_enabled",
|
||||||
@@ -338,23 +349,56 @@ def plan_configuration_change(
|
|||||||
) -> ConfigurationChangeSafetyPlan:
|
) -> ConfigurationChangeSafetyPlan:
|
||||||
field = classify_configuration_field(key)
|
field = classify_configuration_field(key)
|
||||||
if field is None:
|
if field is None:
|
||||||
return ConfigurationChangeSafetyPlan(
|
return _unknown_configuration_plan(key)
|
||||||
key=key,
|
|
||||||
allowed=False,
|
|
||||||
field=None,
|
|
||||||
blockers=("unknown_configuration_field",),
|
|
||||||
policy_explanation="This setting is not in the configuration safety catalog.",
|
|
||||||
)
|
|
||||||
if not include_env_only and not field.ui_managed:
|
if not include_env_only and not field.ui_managed:
|
||||||
return ConfigurationChangeSafetyPlan(
|
return _deployment_managed_configuration_plan(key, field)
|
||||||
key=key,
|
state = _configuration_change_safety_state(
|
||||||
allowed=False,
|
field,
|
||||||
field=field,
|
actor_scopes=actor_scopes,
|
||||||
risk=field.risk,
|
value=value,
|
||||||
secret_handling=field.secret_handling,
|
dry_run=dry_run,
|
||||||
blockers=("deployment_managed",),
|
maintenance_mode=maintenance_mode,
|
||||||
policy_explanation="This setting is deployment-managed and cannot be edited through the UI.",
|
approval_count=approval_count,
|
||||||
)
|
)
|
||||||
|
return _configuration_change_safety_plan(
|
||||||
|
field,
|
||||||
|
state=state,
|
||||||
|
dry_run=dry_run,
|
||||||
|
maintenance_mode=maintenance_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _unknown_configuration_plan(key: str) -> ConfigurationChangeSafetyPlan:
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=key,
|
||||||
|
allowed=False,
|
||||||
|
field=None,
|
||||||
|
blockers=("unknown_configuration_field",),
|
||||||
|
policy_explanation="This setting is not in the configuration safety catalog.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _deployment_managed_configuration_plan(key: str, field: ConfigurationFieldSafety) -> ConfigurationChangeSafetyPlan:
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=key,
|
||||||
|
allowed=False,
|
||||||
|
field=field,
|
||||||
|
risk=field.risk,
|
||||||
|
secret_handling=field.secret_handling,
|
||||||
|
blockers=("deployment_managed",),
|
||||||
|
policy_explanation="This setting is deployment-managed and cannot be edited through the UI.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_change_safety_state(
|
||||||
|
field: ConfigurationFieldSafety,
|
||||||
|
*,
|
||||||
|
actor_scopes: tuple[str, ...] | list[str],
|
||||||
|
value: object,
|
||||||
|
dry_run: bool,
|
||||||
|
maintenance_mode: bool,
|
||||||
|
approval_count: int,
|
||||||
|
) -> _ConfigurationChangeSafetyState:
|
||||||
blockers: list[str] = []
|
blockers: list[str] = []
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope))
|
missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope))
|
||||||
@@ -376,24 +420,41 @@ def plan_configuration_change(
|
|||||||
blockers.append("env_only_secret")
|
blockers.append("env_only_secret")
|
||||||
if field.rollback_history_required:
|
if field.rollback_history_required:
|
||||||
warnings.append("rollback_history_required")
|
warnings.append("rollback_history_required")
|
||||||
return ConfigurationChangeSafetyPlan(
|
return _ConfigurationChangeSafetyState(
|
||||||
key=field.key,
|
|
||||||
allowed=not blockers,
|
|
||||||
field=field,
|
field=field,
|
||||||
risk=field.risk,
|
|
||||||
missing_scopes=missing_scopes,
|
missing_scopes=missing_scopes,
|
||||||
dry_run_required=field.dry_run_required,
|
blockers=tuple(dict.fromkeys(blockers)),
|
||||||
dry_run_satisfied=not field.dry_run_required or dry_run,
|
warnings=tuple(dict.fromkeys(warnings)),
|
||||||
approval_required=approval_required,
|
approval_required=approval_required,
|
||||||
approval_satisfied=approval_satisfied,
|
approval_satisfied=approval_satisfied,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_change_safety_plan(
|
||||||
|
field: ConfigurationFieldSafety,
|
||||||
|
*,
|
||||||
|
state: _ConfigurationChangeSafetyState,
|
||||||
|
dry_run: bool,
|
||||||
|
maintenance_mode: bool,
|
||||||
|
) -> ConfigurationChangeSafetyPlan:
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=field.key,
|
||||||
|
allowed=not state.blockers,
|
||||||
|
field=field,
|
||||||
|
risk=field.risk,
|
||||||
|
missing_scopes=state.missing_scopes,
|
||||||
|
dry_run_required=field.dry_run_required,
|
||||||
|
dry_run_satisfied=not field.dry_run_required or dry_run,
|
||||||
|
approval_required=state.approval_required,
|
||||||
|
approval_satisfied=state.approval_satisfied,
|
||||||
maintenance_required=field.maintenance_required,
|
maintenance_required=field.maintenance_required,
|
||||||
maintenance_satisfied=not field.maintenance_required or maintenance_mode,
|
maintenance_satisfied=not field.maintenance_required or maintenance_mode,
|
||||||
rollback_history_required=field.rollback_history_required,
|
rollback_history_required=field.rollback_history_required,
|
||||||
secret_handling=field.secret_handling,
|
secret_handling=field.secret_handling,
|
||||||
audit_event=field.audit_event,
|
audit_event=field.audit_event,
|
||||||
policy_explanation=_policy_explanation(field),
|
policy_explanation=_policy_explanation(field),
|
||||||
blockers=tuple(dict.fromkeys(blockers)),
|
blockers=state.blockers,
|
||||||
warnings=tuple(dict.fromkeys(warnings)),
|
warnings=state.warnings,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -411,13 +472,4 @@ def _policy_explanation(field: ConfigurationFieldSafety) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _contains_plain_secret(value: object) -> bool:
|
def _contains_plain_secret(value: object) -> bool:
|
||||||
if not isinstance(value, Mapping):
|
return contains_plain_secret(value)
|
||||||
return False
|
|
||||||
secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
|
|
||||||
for key, item in value.items():
|
|
||||||
name = str(key).strip().casefold()
|
|
||||||
if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
|
|
||||||
return True
|
|
||||||
if isinstance(item, Mapping) and _contains_plain_secret(item):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|||||||
13
src/govoplan_core/core/files.py
Normal file
13
src/govoplan_core/core/files.py
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
|
from govoplan_core.core.access import ResourceAccessExplanationProvider
|
||||||
|
|
||||||
|
|
||||||
|
CAPABILITY_FILES_ACCESS = "files.access"
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class FileAccessProvider(ResourceAccessExplanationProvider, Protocol):
|
||||||
|
"""Resource-level access explanation provider for Files-owned resources."""
|
||||||
@@ -72,6 +72,25 @@ class ConfigValidationResult:
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _RuntimeProfile:
|
||||||
|
name: str
|
||||||
|
production: bool
|
||||||
|
production_like: bool
|
||||||
|
local: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class _ConfigIssueCollector:
|
||||||
|
strict: bool
|
||||||
|
issues: list[ConfigIssue]
|
||||||
|
|
||||||
|
def add(self, level: ConfigIssueLevel, key: str, message: str, action: str) -> None:
|
||||||
|
if self.strict and level == "warning":
|
||||||
|
level = "error"
|
||||||
|
self.issues.append(ConfigIssue(level=level, key=key, message=message, action=action))
|
||||||
|
|
||||||
|
|
||||||
_LOCAL_PROFILES = {"dev", "local", "local-dev", "test"}
|
_LOCAL_PROFILES = {"dev", "local", "local-dev", "test"}
|
||||||
_PRODUCTION_PROFILES = {"prod", "production", "self-hosted"}
|
_PRODUCTION_PROFILES = {"prod", "production", "self-hosted"}
|
||||||
_PRODUCTION_LIKE_PROFILES = {"staging", "production-like", "production-like-dev"}
|
_PRODUCTION_LIKE_PROFILES = {"staging", "production-like", "production-like-dev"}
|
||||||
@@ -114,95 +133,130 @@ def validate_runtime_configuration(
|
|||||||
strict: bool = False,
|
strict: bool = False,
|
||||||
) -> ConfigValidationResult:
|
) -> ConfigValidationResult:
|
||||||
env = dict(os.environ if environ is None else environ)
|
env = dict(os.environ if environ is None else environ)
|
||||||
clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV"))
|
runtime = _runtime_profile(env, profile=profile)
|
||||||
issues: list[ConfigIssue] = []
|
collector = _ConfigIssueCollector(strict=strict, issues=[])
|
||||||
|
_validate_app_env(env, runtime, collector)
|
||||||
|
_validate_database_settings(env, runtime, collector)
|
||||||
|
_validate_master_key(env, runtime, collector)
|
||||||
|
_validate_enabled_modules(env, runtime, collector)
|
||||||
|
_validate_async_and_auth_settings(env, runtime, collector)
|
||||||
|
_validate_cors_settings(env, runtime, collector)
|
||||||
|
_validate_file_storage_settings(env, runtime, collector)
|
||||||
|
_validate_module_catalog_trust(env, runtime, collector)
|
||||||
|
return ConfigValidationResult(profile=runtime.name, issues=tuple(collector.issues))
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _RuntimeProfile:
|
||||||
|
clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV"))
|
||||||
production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"}
|
production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"}
|
||||||
production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES
|
production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES
|
||||||
local = clean_profile in _LOCAL_PROFILES and not production_like
|
return _RuntimeProfile(
|
||||||
|
name=clean_profile,
|
||||||
|
production=production,
|
||||||
|
production_like=production_like,
|
||||||
|
local=clean_profile in _LOCAL_PROFILES and not production_like,
|
||||||
|
)
|
||||||
|
|
||||||
def issue(level: ConfigIssueLevel, key: str, message: str, action: str) -> None:
|
|
||||||
if strict and level == "warning":
|
|
||||||
level = "error"
|
|
||||||
issues.append(ConfigIssue(level=level, key=key, message=message, action=action))
|
|
||||||
|
|
||||||
|
def _validate_app_env(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
app_env = _clean(env.get("APP_ENV"))
|
app_env = _clean(env.get("APP_ENV"))
|
||||||
if not app_env and production_like:
|
if not app_env and runtime.production_like:
|
||||||
issue("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.")
|
collector.add("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.")
|
||||||
elif app_env.lower() in {"dev", "test", "local"} and production_like:
|
elif app_env.lower() in {"dev", "test", "local"} and runtime.production_like:
|
||||||
issue("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {clean_profile!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.")
|
collector.add("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_database_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
database_url = _clean(env.get("DATABASE_URL"))
|
database_url = _clean(env.get("DATABASE_URL"))
|
||||||
if not database_url:
|
if not database_url:
|
||||||
issue("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.")
|
collector.add("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.")
|
||||||
else:
|
return
|
||||||
backend = _database_backend(database_url)
|
backend = _database_backend(database_url)
|
||||||
if backend is None:
|
if backend is None:
|
||||||
issue("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.")
|
collector.add("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.")
|
||||||
elif backend == "sqlite" and production_like:
|
return
|
||||||
issue("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.")
|
if backend == "sqlite" and runtime.production_like:
|
||||||
elif backend != "postgresql" and production:
|
collector.add("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.")
|
||||||
issue("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.")
|
elif backend != "postgresql" and runtime.production:
|
||||||
if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")):
|
collector.add("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.")
|
||||||
issue("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.")
|
if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")):
|
||||||
|
collector.add("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_master_key(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
master_key = _clean(env.get("MASTER_KEY_B64"))
|
master_key = _clean(env.get("MASTER_KEY_B64"))
|
||||||
if not master_key and not local:
|
if not master_key and not runtime.local:
|
||||||
issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.")
|
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.")
|
||||||
elif master_key:
|
return
|
||||||
error = _master_key_error(master_key)
|
if not master_key:
|
||||||
if error:
|
return
|
||||||
issue("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.")
|
error = _master_key_error(master_key)
|
||||||
elif "change-me" in master_key.lower() or "generate" in master_key.lower():
|
if error:
|
||||||
issue("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.")
|
collector.add("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.")
|
||||||
|
elif "change-me" in master_key.lower() or "generate" in master_key.lower():
|
||||||
|
collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_enabled_modules(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
enabled_modules = _csv(env.get("ENABLED_MODULES"))
|
enabled_modules = _csv(env.get("ENABLED_MODULES"))
|
||||||
if not enabled_modules and production_like:
|
if not enabled_modules and runtime.production_like:
|
||||||
issue("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.")
|
collector.add("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.")
|
||||||
elif "access" not in enabled_modules and production_like:
|
elif "access" not in enabled_modules and runtime.production_like:
|
||||||
issue("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.")
|
collector.add("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.")
|
||||||
elif enabled_modules and "admin" not in enabled_modules and production_like:
|
elif enabled_modules and "admin" not in enabled_modules and runtime.production_like:
|
||||||
issue("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.")
|
collector.add("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.")
|
||||||
|
|
||||||
celery_enabled = _truthy(env.get("CELERY_ENABLED"))
|
|
||||||
if celery_enabled and not _clean(env.get("REDIS_URL")):
|
|
||||||
issue("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.")
|
|
||||||
|
|
||||||
if production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")):
|
def _validate_async_and_auth_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
issue("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.")
|
if _truthy(env.get("CELERY_ENABLED")) and not _clean(env.get("REDIS_URL")):
|
||||||
|
collector.add("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.")
|
||||||
|
if runtime.production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")):
|
||||||
|
collector.add("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.")
|
||||||
|
if runtime.production and not _truthy(env.get("AUTH_COOKIE_SECURE")):
|
||||||
|
collector.add("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.")
|
||||||
|
|
||||||
if production and not _truthy(env.get("AUTH_COOKIE_SECURE")):
|
|
||||||
issue("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.")
|
|
||||||
|
|
||||||
|
def _validate_cors_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
cors_origins = _csv(env.get("CORS_ORIGINS"))
|
cors_origins = _csv(env.get("CORS_ORIGINS"))
|
||||||
if production_like and not cors_origins:
|
if runtime.production_like and not cors_origins:
|
||||||
issue("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.")
|
collector.add("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.")
|
||||||
elif "*" in cors_origins and production_like:
|
elif "*" in cors_origins and runtime.production_like:
|
||||||
issue("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.")
|
collector.add("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.")
|
||||||
elif production and set(cors_origins) <= _DEFAULT_LOCAL_CORS:
|
elif runtime.production and set(cors_origins) <= _DEFAULT_LOCAL_CORS:
|
||||||
issue("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.")
|
collector.add("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local"
|
storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local"
|
||||||
if storage_backend == "local":
|
if storage_backend == "local":
|
||||||
if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and production_like:
|
_validate_local_file_storage(env, runtime, collector)
|
||||||
issue("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.")
|
|
||||||
elif production:
|
|
||||||
issue("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.")
|
|
||||||
elif storage_backend == "s3":
|
elif storage_backend == "s3":
|
||||||
for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"):
|
_validate_s3_file_storage(env, collector)
|
||||||
if not _clean(env.get(key)):
|
|
||||||
issue("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.")
|
|
||||||
else:
|
else:
|
||||||
issue("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.")
|
collector.add("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_local_file_storage(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
|
if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and runtime.production_like:
|
||||||
|
collector.add("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.")
|
||||||
|
elif runtime.production:
|
||||||
|
collector.add("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_s3_file_storage(env: Mapping[str, str], collector: _ConfigIssueCollector) -> None:
|
||||||
|
for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"):
|
||||||
|
if not _clean(env.get(key)):
|
||||||
|
collector.add("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_module_catalog_trust(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None:
|
||||||
catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG"))
|
catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG"))
|
||||||
if production and catalog_source:
|
if not runtime.production or not catalog_source:
|
||||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")):
|
return
|
||||||
issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.")
|
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")):
|
||||||
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")):
|
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.")
|
||||||
issue("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.")
|
if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")):
|
||||||
|
collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.")
|
||||||
return ConfigValidationResult(profile=clean_profile, issues=tuple(issues))
|
|
||||||
|
|
||||||
|
|
||||||
def _self_hosted_env_template(master_key: str) -> str:
|
def _self_hosted_env_template(master_key: str) -> str:
|
||||||
@@ -299,8 +353,8 @@ def _master_key_error(value: str) -> str | None:
|
|||||||
try:
|
try:
|
||||||
Fernet(candidate)
|
Fernet(candidate)
|
||||||
return None
|
return None
|
||||||
except Exception:
|
except (TypeError, ValueError):
|
||||||
pass
|
raw = None
|
||||||
try:
|
try:
|
||||||
raw = base64.b64decode(candidate)
|
raw = base64.b64decode(candidate)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -309,6 +363,6 @@ def _master_key_error(value: str) -> str | None:
|
|||||||
return "MASTER_KEY_B64 must decode to exactly 32 bytes."
|
return "MASTER_KEY_B64 must decode to exactly 32 bytes."
|
||||||
try:
|
try:
|
||||||
Fernet(base64.urlsafe_b64encode(raw))
|
Fernet(base64.urlsafe_b64encode(raw))
|
||||||
except Exception:
|
except (TypeError, ValueError):
|
||||||
return "MASTER_KEY_B64 is not usable as a Fernet key."
|
return "MASTER_KEY_B64 is not usable as a Fernet key."
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -12,6 +12,29 @@ from govoplan_core.core.registry import PlatformRegistry
|
|||||||
class MigrationMetadataPlan:
|
class MigrationMetadataPlan:
|
||||||
metadata: tuple[MetaData, ...]
|
metadata: tuple[MetaData, ...]
|
||||||
script_locations: tuple[str, ...]
|
script_locations: tuple[str, ...]
|
||||||
|
modules: tuple[ModuleMigrationMetadata, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ModuleMigrationMetadata:
|
||||||
|
module_id: str
|
||||||
|
script_location: str | None = None
|
||||||
|
has_metadata: bool = False
|
||||||
|
migration_after: tuple[str, ...] = ()
|
||||||
|
migration_before: tuple[str, ...] = ()
|
||||||
|
migration_tasks: tuple[ModuleMigrationTaskMetadata, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ModuleMigrationTaskMetadata:
|
||||||
|
task_id: str
|
||||||
|
phase: str
|
||||||
|
summary: str
|
||||||
|
task_version: str = "1"
|
||||||
|
safety: str = "automatic"
|
||||||
|
idempotent: bool = True
|
||||||
|
timeout_seconds: int | None = None
|
||||||
|
has_executor: bool = False
|
||||||
|
|
||||||
|
|
||||||
def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan:
|
def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan:
|
||||||
@@ -28,4 +51,28 @@ def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Itera
|
|||||||
metadata.append(spec.metadata)
|
metadata.append(spec.metadata)
|
||||||
if spec.script_location:
|
if spec.script_location:
|
||||||
script_locations.append(spec.script_location)
|
script_locations.append(spec.script_location)
|
||||||
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations))
|
modules = tuple(
|
||||||
|
ModuleMigrationMetadata(
|
||||||
|
module_id=manifest.id,
|
||||||
|
script_location=manifest.migration_spec.script_location,
|
||||||
|
has_metadata=isinstance(manifest.migration_spec.metadata, MetaData),
|
||||||
|
migration_after=tuple(manifest.migration_spec.migration_after),
|
||||||
|
migration_before=tuple(manifest.migration_spec.migration_before),
|
||||||
|
migration_tasks=tuple(
|
||||||
|
ModuleMigrationTaskMetadata(
|
||||||
|
task_id=task.task_id,
|
||||||
|
phase=task.phase,
|
||||||
|
summary=task.summary,
|
||||||
|
task_version=task.task_version,
|
||||||
|
safety=task.safety,
|
||||||
|
idempotent=task.idempotent,
|
||||||
|
timeout_seconds=task.timeout_seconds,
|
||||||
|
has_executor=task.executor is not None,
|
||||||
|
)
|
||||||
|
for task in manifest.migration_spec.migration_tasks
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for manifest in registry.manifests()
|
||||||
|
if manifest.migration_spec is not None
|
||||||
|
)
|
||||||
|
return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations), modules=modules)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
125
src/govoplan_core/core/module_installer_notifications.py
Normal file
125
src/govoplan_core/core/module_installer_notifications.py
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||||
|
from govoplan_core.core.modules import ModuleContext
|
||||||
|
from govoplan_core.core.notifications import NotificationDispatchRequest, notification_dispatch_provider
|
||||||
|
from govoplan_core.server.registry import available_module_manifests, build_platform_registry
|
||||||
|
|
||||||
|
|
||||||
|
INSTALLER_NOTIFICATION_ACTION_URL = "/admin?section=modules"
|
||||||
|
|
||||||
|
|
||||||
|
def emit_module_installer_notification(
|
||||||
|
*,
|
||||||
|
session: object,
|
||||||
|
registry: object | None,
|
||||||
|
tenant_id: str | None,
|
||||||
|
request: Mapping[str, object],
|
||||||
|
event_kind: str,
|
||||||
|
subject: str,
|
||||||
|
body_text: str,
|
||||||
|
recipient_id: str | None = None,
|
||||||
|
priority: int = 5,
|
||||||
|
) -> bool:
|
||||||
|
if not tenant_id:
|
||||||
|
return False
|
||||||
|
provider = notification_dispatch_provider(registry)
|
||||||
|
if provider is None:
|
||||||
|
return False
|
||||||
|
request_id = str(request.get("request_id") or "")
|
||||||
|
if not request_id:
|
||||||
|
return False
|
||||||
|
provider.enqueue_notification(
|
||||||
|
session,
|
||||||
|
NotificationDispatchRequest(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
source_module="core",
|
||||||
|
source_resource_type="module_install_request",
|
||||||
|
source_resource_id=request_id,
|
||||||
|
event_kind=event_kind,
|
||||||
|
channel="inbox",
|
||||||
|
recipient_type="user" if recipient_id else None,
|
||||||
|
recipient_id=recipient_id,
|
||||||
|
subject=subject,
|
||||||
|
body_text=body_text,
|
||||||
|
action_url=INSTALLER_NOTIFICATION_ACTION_URL,
|
||||||
|
priority=priority,
|
||||||
|
payload={
|
||||||
|
"request_id": request_id,
|
||||||
|
"status": request.get("status"),
|
||||||
|
"run_id": _result_run_id(request),
|
||||||
|
},
|
||||||
|
metadata={
|
||||||
|
"trace": request.get("trace") if isinstance(request.get("trace"), Mapping) else None,
|
||||||
|
"retry_of": request.get("retry_of"),
|
||||||
|
},
|
||||||
|
),
|
||||||
|
enqueue_delivery=False,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def build_runtime_notification_registry(settings: object) -> object | None:
|
||||||
|
try:
|
||||||
|
raw_enabled_modules = load_startup_enabled_modules(str(getattr(settings, "enabled_modules", "") or ""))
|
||||||
|
candidate_modules = startup_candidate_module_ids(str(getattr(settings, "enabled_modules", "") or ""), raw_enabled_modules)
|
||||||
|
available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True)
|
||||||
|
enabled_modules = load_startup_enabled_modules(str(getattr(settings, "enabled_modules", "") or ""), available=available_modules)
|
||||||
|
if "notifications" not in enabled_modules:
|
||||||
|
return None
|
||||||
|
registry = build_platform_registry(enabled_modules)
|
||||||
|
registry.configure_capability_context(ModuleContext(registry=registry, settings=settings))
|
||||||
|
return registry
|
||||||
|
except Exception: # noqa: BLE001 - notification bridge must not block installer work.
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def installer_notification_subject(event_kind: str, request: Mapping[str, object]) -> str:
|
||||||
|
request_id = str(request.get("request_id") or "unknown")
|
||||||
|
status = str(request.get("status") or event_kind.rsplit(".", 1)[-1])
|
||||||
|
prefixes = {
|
||||||
|
"queued": "Module installer request queued",
|
||||||
|
"running": "Module installer request started",
|
||||||
|
"completed": "Module installer request completed",
|
||||||
|
"failed": "Module installer request failed",
|
||||||
|
"cancelled": "Module installer request cancelled",
|
||||||
|
}
|
||||||
|
prefix = prefixes.get(status, "Module installer request updated")
|
||||||
|
return ": ".join((prefix, request_id))
|
||||||
|
|
||||||
|
|
||||||
|
def _installer_status_sentence(request_id: str, status: str) -> str:
|
||||||
|
return " ".join(("Installer request", request_id, "is", status))
|
||||||
|
|
||||||
|
|
||||||
|
def installer_notification_body(event_kind: str, request: Mapping[str, object]) -> str:
|
||||||
|
status = str(request.get("status") or event_kind.rsplit(".", 1)[-1])
|
||||||
|
request_id = str(request.get("request_id") or "unknown")
|
||||||
|
result = request.get("result") if isinstance(request.get("result"), Mapping) else {}
|
||||||
|
error = str(request.get("error") or result.get("error") or "").strip()
|
||||||
|
sentence = _installer_status_sentence(request_id, status)
|
||||||
|
if error:
|
||||||
|
return ". Error: ".join((sentence, error))
|
||||||
|
run_id = _result_run_id(request)
|
||||||
|
if run_id:
|
||||||
|
return ". Run: ".join((sentence, run_id))
|
||||||
|
return sentence + "."
|
||||||
|
|
||||||
|
|
||||||
|
def installer_notification_priority(status: str) -> int:
|
||||||
|
if status == "failed":
|
||||||
|
return 20
|
||||||
|
if status in {"completed", "cancelled"}:
|
||||||
|
return 8
|
||||||
|
if status == "running":
|
||||||
|
return 4
|
||||||
|
return 5
|
||||||
|
|
||||||
|
|
||||||
|
def _result_run_id(request: Mapping[str, object]) -> str | None:
|
||||||
|
result = request.get("result") if isinstance(request.get("result"), Mapping) else {}
|
||||||
|
run_id: Any = result.get("run_id") if isinstance(result, Mapping) else None
|
||||||
|
return str(run_id) if run_id else None
|
||||||
@@ -7,12 +7,11 @@ import json
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
from cryptography.exceptions import InvalidSignature
|
from cryptography.exceptions import InvalidSignature
|
||||||
from cryptography.hazmat.primitives import serialization
|
from cryptography.hazmat.primitives import serialization
|
||||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http_text
|
||||||
|
|
||||||
|
|
||||||
def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]:
|
def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]:
|
||||||
@@ -257,17 +256,14 @@ def _configured_trusted_keys_cache_path() -> Path | None:
|
|||||||
|
|
||||||
|
|
||||||
def _read_trusted_keys_url(url: str) -> str:
|
def _read_trusted_keys_url(url: str) -> str:
|
||||||
if not url.startswith(("https://", "http://")):
|
|
||||||
raise ValueError("Trusted license key URL must use http:// or https://.")
|
|
||||||
cache_path = _configured_trusted_keys_cache_path()
|
cache_path = _configured_trusted_keys_cache_path()
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=15) as response:
|
body = fetch_http_text(url, timeout=15, label="Trusted license key URL")
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return body
|
return body
|
||||||
except (OSError, urllib.error.URLError):
|
except OSError:
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
return cache_path.read_text(encoding="utf-8")
|
return cache_path.read_text(encoding="utf-8")
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ MODULE_SETTINGS_KEY = "module_management"
|
|||||||
INSTALL_PLAN_KEY = "install_plan"
|
INSTALL_PLAN_KEY = "install_plan"
|
||||||
REQUIRED_PLATFORM_MODULES = ("access",)
|
REQUIRED_PLATFORM_MODULES = ("access",)
|
||||||
PROTECTED_MODULES = (*REQUIRED_PLATFORM_MODULES, "admin")
|
PROTECTED_MODULES = (*REQUIRED_PLATFORM_MODULES, "admin")
|
||||||
INSTALL_PLAN_ACTIONS = ("install", "uninstall")
|
INSTALL_PLAN_ACTIONS = ("install", "update", "uninstall")
|
||||||
INSTALL_PLAN_STATUSES = ("planned", "applied", "blocked")
|
INSTALL_PLAN_STATUSES = ("planned", "applied", "blocked")
|
||||||
INSTALL_PLAN_SOURCES = ("manual", "catalog")
|
INSTALL_PLAN_SOURCES = ("manual", "catalog")
|
||||||
LOCAL_DEPENDENCY_REF_PREFIXES = ("file:", "path:", "workspace:", "link:")
|
LOCAL_DEPENDENCY_REF_PREFIXES = ("file:", "path:", "workspace:", "link:")
|
||||||
@@ -46,6 +46,7 @@ class ModuleInstallPlanItem:
|
|||||||
webui_package: str | None = None
|
webui_package: str | None = None
|
||||||
webui_ref: str | None = None
|
webui_ref: str | None = None
|
||||||
artifact_integrity: Mapping[str, object] | None = None
|
artifact_integrity: Mapping[str, object] | None = None
|
||||||
|
data_safety_acknowledged: bool = False
|
||||||
destroy_data: bool = False
|
destroy_data: bool = False
|
||||||
status: str = "planned"
|
status: str = "planned"
|
||||||
notes: str | None = None
|
notes: str | None = None
|
||||||
@@ -59,6 +60,8 @@ class ModuleInstallPlanItem:
|
|||||||
}
|
}
|
||||||
if self.destroy_data:
|
if self.destroy_data:
|
||||||
payload["destroy_data"] = True
|
payload["destroy_data"] = True
|
||||||
|
if self.data_safety_acknowledged:
|
||||||
|
payload["data_safety_acknowledged"] = True
|
||||||
if self.catalog:
|
if self.catalog:
|
||||||
payload["catalog"] = dict(self.catalog)
|
payload["catalog"] = dict(self.catalog)
|
||||||
for key in ("python_package", "python_ref", "webui_package", "webui_ref", "notes"):
|
for key in ("python_package", "python_ref", "webui_package", "webui_ref", "notes"):
|
||||||
@@ -70,6 +73,23 @@ class ModuleInstallPlanItem:
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _NormalizedModuleInstallPlanItem:
|
||||||
|
module_id: str
|
||||||
|
action: str
|
||||||
|
source: str
|
||||||
|
catalog: Mapping[str, object] | None
|
||||||
|
status: str
|
||||||
|
python_package: str | None
|
||||||
|
python_ref: str | None
|
||||||
|
webui_package: str | None
|
||||||
|
webui_ref: str | None
|
||||||
|
artifact_integrity: Mapping[str, object] | None
|
||||||
|
data_safety_acknowledged: bool
|
||||||
|
destroy_data: bool
|
||||||
|
notes: str | None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class ModuleInstallPlan:
|
class ModuleInstallPlan:
|
||||||
items: tuple[ModuleInstallPlanItem, ...] = ()
|
items: tuple[ModuleInstallPlanItem, ...] = ()
|
||||||
@@ -206,7 +226,7 @@ def module_install_plan_commands(
|
|||||||
for item in items:
|
for item in items:
|
||||||
if item.status not in included_statuses:
|
if item.status not in included_statuses:
|
||||||
continue
|
continue
|
||||||
if item.action == "install":
|
if item.action in {"install", "update"}:
|
||||||
if item.python_ref:
|
if item.python_ref:
|
||||||
commands.append(f"python -m pip install {shlex.quote(item.python_ref)}")
|
commands.append(f"python -m pip install {shlex.quote(item.python_ref)}")
|
||||||
if item.webui_package and item.webui_ref:
|
if item.webui_package and item.webui_ref:
|
||||||
@@ -281,13 +301,35 @@ def desired_modules_after_package_plan(
|
|||||||
removed = {item.module_id for item in planned_items if item.action == "uninstall"}
|
removed = {item.module_id for item in planned_items if item.action == "uninstall"}
|
||||||
desired = [module_id for module_id in desired if module_id not in removed]
|
desired = [module_id for module_id in desired if module_id not in removed]
|
||||||
if activate_installed_modules:
|
if activate_installed_modules:
|
||||||
desired.extend(item.module_id for item in planned_items if item.action == "install")
|
desired.extend(item.module_id for item in planned_items if item.action in {"install", "update"})
|
||||||
return tuple(dict.fromkeys(desired))
|
return tuple(dict.fromkeys(desired))
|
||||||
|
|
||||||
|
|
||||||
def normalize_module_install_plan_item(
|
def normalize_module_install_plan_item(
|
||||||
item: Mapping[str, object] | ModuleInstallPlanItem,
|
item: Mapping[str, object] | ModuleInstallPlanItem,
|
||||||
) -> ModuleInstallPlanItem:
|
) -> ModuleInstallPlanItem:
|
||||||
|
normalized = _normalized_module_install_plan_item(item)
|
||||||
|
_validate_module_install_plan_item(normalized)
|
||||||
|
return ModuleInstallPlanItem(
|
||||||
|
module_id=normalized.module_id,
|
||||||
|
action=normalized.action,
|
||||||
|
source=normalized.source,
|
||||||
|
catalog=normalized.catalog,
|
||||||
|
python_package=normalized.python_package,
|
||||||
|
python_ref=normalized.python_ref,
|
||||||
|
webui_package=normalized.webui_package,
|
||||||
|
webui_ref=normalized.webui_ref,
|
||||||
|
artifact_integrity=normalized.artifact_integrity,
|
||||||
|
data_safety_acknowledged=normalized.data_safety_acknowledged,
|
||||||
|
destroy_data=normalized.destroy_data,
|
||||||
|
status=normalized.status,
|
||||||
|
notes=normalized.notes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_module_install_plan_item(
|
||||||
|
item: Mapping[str, object] | ModuleInstallPlanItem,
|
||||||
|
) -> _NormalizedModuleInstallPlanItem:
|
||||||
if isinstance(item, ModuleInstallPlanItem):
|
if isinstance(item, ModuleInstallPlanItem):
|
||||||
raw = item.as_dict()
|
raw = item.as_dict()
|
||||||
elif isinstance(item, Mapping):
|
elif isinstance(item, Mapping):
|
||||||
@@ -305,46 +347,57 @@ def normalize_module_install_plan_item(
|
|||||||
webui_package = _clean_optional_string(raw.get("webui_package"))
|
webui_package = _clean_optional_string(raw.get("webui_package"))
|
||||||
webui_ref = _clean_optional_string(raw.get("webui_ref"))
|
webui_ref = _clean_optional_string(raw.get("webui_ref"))
|
||||||
artifact_integrity = _clean_optional_mapping(raw.get("artifact_integrity"), field="artifact_integrity", module_id=module_id)
|
artifact_integrity = _clean_optional_mapping(raw.get("artifact_integrity"), field="artifact_integrity", module_id=module_id)
|
||||||
|
data_safety_acknowledged = _clean_bool(raw.get("data_safety_acknowledged"))
|
||||||
destroy_data = _clean_bool(raw.get("destroy_data"))
|
destroy_data = _clean_bool(raw.get("destroy_data"))
|
||||||
notes = _clean_optional_string(raw.get("notes"))
|
notes = _clean_optional_string(raw.get("notes"))
|
||||||
|
return _NormalizedModuleInstallPlanItem(
|
||||||
if action not in INSTALL_PLAN_ACTIONS:
|
|
||||||
raise ModuleManagementError(f"Unsupported install plan action for {module_id!r}: {action!r}.")
|
|
||||||
if status not in INSTALL_PLAN_STATUSES:
|
|
||||||
raise ModuleManagementError(f"Unsupported install plan status for {module_id!r}: {status!r}.")
|
|
||||||
if source not in INSTALL_PLAN_SOURCES:
|
|
||||||
raise ModuleManagementError(f"Unsupported install plan source for {module_id!r}: {source!r}.")
|
|
||||||
if action == "install" and not python_ref:
|
|
||||||
raise ModuleManagementError(f"Install plan item {module_id!r} needs a Python package reference.")
|
|
||||||
if action == "uninstall" and not python_package:
|
|
||||||
raise ModuleManagementError(f"Uninstall plan item {module_id!r} needs a Python package name.")
|
|
||||||
if action != "uninstall" and destroy_data:
|
|
||||||
raise ModuleManagementError(f"Install plan item {module_id!r} can only destroy data during uninstall.")
|
|
||||||
if action == "install" and bool(webui_package) != bool(webui_ref):
|
|
||||||
raise ModuleManagementError(f"Install plan item {module_id!r} needs both WebUI package and WebUI reference, or neither.")
|
|
||||||
if action == "uninstall" and webui_ref and not webui_package:
|
|
||||||
raise ModuleManagementError(f"Uninstall plan item {module_id!r} has a WebUI reference but no WebUI package.")
|
|
||||||
if python_ref:
|
|
||||||
_validate_dependency_ref(python_ref, field="python_ref", module_id=module_id)
|
|
||||||
if webui_ref:
|
|
||||||
_validate_dependency_ref(webui_ref, field="webui_ref", module_id=module_id)
|
|
||||||
|
|
||||||
return ModuleInstallPlanItem(
|
|
||||||
module_id=module_id,
|
module_id=module_id,
|
||||||
action=action,
|
action=action,
|
||||||
source=source,
|
source=source,
|
||||||
catalog=catalog,
|
catalog=catalog,
|
||||||
|
status=status,
|
||||||
python_package=python_package,
|
python_package=python_package,
|
||||||
python_ref=python_ref,
|
python_ref=python_ref,
|
||||||
webui_package=webui_package,
|
webui_package=webui_package,
|
||||||
webui_ref=webui_ref,
|
webui_ref=webui_ref,
|
||||||
artifact_integrity=artifact_integrity,
|
artifact_integrity=artifact_integrity,
|
||||||
|
data_safety_acknowledged=data_safety_acknowledged,
|
||||||
destroy_data=destroy_data,
|
destroy_data=destroy_data,
|
||||||
status=status,
|
|
||||||
notes=notes,
|
notes=notes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_module_install_plan_item(item: _NormalizedModuleInstallPlanItem) -> None:
|
||||||
|
if item.action not in INSTALL_PLAN_ACTIONS:
|
||||||
|
raise ModuleManagementError(f"Unsupported install plan action for {item.module_id!r}: {item.action!r}.")
|
||||||
|
if item.status not in INSTALL_PLAN_STATUSES:
|
||||||
|
raise ModuleManagementError(f"Unsupported install plan status for {item.module_id!r}: {item.status!r}.")
|
||||||
|
if item.source not in INSTALL_PLAN_SOURCES:
|
||||||
|
raise ModuleManagementError(f"Unsupported install plan source for {item.module_id!r}: {item.source!r}.")
|
||||||
|
_validate_module_install_plan_item_requirements(item)
|
||||||
|
_validate_module_install_plan_item_refs(item)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_module_install_plan_item_requirements(item: _NormalizedModuleInstallPlanItem) -> None:
|
||||||
|
if item.action in {"install", "update"} and not item.python_ref:
|
||||||
|
raise ModuleManagementError(f"Install plan item {item.module_id!r} needs a Python package reference.")
|
||||||
|
if item.action == "uninstall" and not item.python_package:
|
||||||
|
raise ModuleManagementError(f"Uninstall plan item {item.module_id!r} needs a Python package name.")
|
||||||
|
if item.action != "uninstall" and item.destroy_data:
|
||||||
|
raise ModuleManagementError(f"Install plan item {item.module_id!r} can only destroy data during uninstall.")
|
||||||
|
if item.action in {"install", "update"} and bool(item.webui_package) != bool(item.webui_ref):
|
||||||
|
raise ModuleManagementError(f"Install plan item {item.module_id!r} needs both WebUI package and WebUI reference, or neither.")
|
||||||
|
if item.action == "uninstall" and item.webui_ref and not item.webui_package:
|
||||||
|
raise ModuleManagementError(f"Uninstall plan item {item.module_id!r} has a WebUI reference but no WebUI package.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_module_install_plan_item_refs(item: _NormalizedModuleInstallPlanItem) -> None:
|
||||||
|
if item.python_ref:
|
||||||
|
_validate_dependency_ref(item.python_ref, field="python_ref", module_id=item.module_id)
|
||||||
|
if item.webui_ref:
|
||||||
|
_validate_dependency_ref(item.webui_ref, field="webui_ref", module_id=item.module_id)
|
||||||
|
|
||||||
|
|
||||||
def plan_desired_enabled_modules(
|
def plan_desired_enabled_modules(
|
||||||
requested_enabled: Iterable[str],
|
requested_enabled: Iterable[str],
|
||||||
available: Mapping[str, ModuleManifest],
|
available: Mapping[str, ModuleManifest],
|
||||||
|
|||||||
@@ -3,22 +3,43 @@ from __future__ import annotations
|
|||||||
import base64
|
import base64
|
||||||
import binascii
|
import binascii
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from typing import Any
|
from typing import Any
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
from cryptography.exceptions import InvalidSignature
|
from cryptography.exceptions import InvalidSignature
|
||||||
from cryptography.hazmat.primitives import serialization
|
from cryptography.hazmat.primitives import serialization
|
||||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
||||||
|
|
||||||
from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range
|
from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range
|
||||||
|
from govoplan_core.security.http_fetch import fetch_http_text, is_http_url
|
||||||
|
|
||||||
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
_INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
|
||||||
|
CATALOG_MIGRATION_SAFETY = ("automatic", "requires_review", "forward_only", "destructive")
|
||||||
|
CATALOG_MIGRATION_TASK_PHASES = (
|
||||||
|
"pre_migration_check",
|
||||||
|
"pre_migration_prepare",
|
||||||
|
"post_migration_backfill",
|
||||||
|
"post_migration_verify",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _CatalogValidationState:
|
||||||
|
modules: tuple[dict[str, object], ...]
|
||||||
|
channel: str | None
|
||||||
|
sequence: int | None
|
||||||
|
generated_at: str | None
|
||||||
|
not_before: str | None
|
||||||
|
expires_at: str | None
|
||||||
|
signature_state: dict[str, object]
|
||||||
|
freshness: dict[str, object]
|
||||||
|
replay: dict[str, object]
|
||||||
|
read_state: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
def module_package_catalog(
|
def module_package_catalog(
|
||||||
@@ -54,178 +75,143 @@ def validate_module_package_catalog(
|
|||||||
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
||||||
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
||||||
if catalog_source is not None and not _catalog_source_exists(catalog_source):
|
if catalog_source is not None and not _catalog_source_exists(catalog_source):
|
||||||
return {
|
return _catalog_error_result(catalog_source, error=f"Module package catalog does not exist: {catalog_source}")
|
||||||
"valid": False,
|
|
||||||
"configured": True,
|
|
||||||
"path": str(catalog_source),
|
|
||||||
"source": str(catalog_source),
|
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
|
||||||
"cache_used": False,
|
|
||||||
"cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None,
|
|
||||||
"modules": [],
|
|
||||||
"channel": None,
|
|
||||||
"sequence": None,
|
|
||||||
"generated_at": None,
|
|
||||||
"not_before": None,
|
|
||||||
"expires_at": None,
|
|
||||||
"signed": False,
|
|
||||||
"trusted": False,
|
|
||||||
"key_id": None,
|
|
||||||
"warnings": [],
|
|
||||||
"error": f"Module package catalog does not exist: {catalog_source}",
|
|
||||||
}
|
|
||||||
warnings: list[str] = []
|
|
||||||
read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None}
|
|
||||||
try:
|
try:
|
||||||
payload, read_state = _read_catalog_payload_with_metadata(catalog_source)
|
state = _catalog_validation_state(catalog_source, trusted_keys=effective_trusted_keys)
|
||||||
modules = _normalize_catalog_modules(payload)
|
|
||||||
channel = _catalog_channel(payload)
|
|
||||||
sequence = _catalog_sequence(payload)
|
|
||||||
generated_at = _catalog_optional_text(payload, "generated_at")
|
|
||||||
not_before = _catalog_optional_text(payload, "not_before")
|
|
||||||
expires_at = _catalog_optional_text(payload, "expires_at")
|
|
||||||
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
|
||||||
freshness = _catalog_freshness_state(payload)
|
|
||||||
replay = _catalog_replay_state(channel=channel, sequence=sequence)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {
|
return _catalog_error_result(catalog_source, error=str(exc))
|
||||||
"valid": False,
|
policy_error = _catalog_policy_error(
|
||||||
"configured": catalog_source is not None,
|
catalog_source,
|
||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
state,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
require_trusted=effective_require_trusted,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
approved_channels=effective_approved_channels,
|
||||||
"cache_used": bool(read_state.get("cache_used")),
|
)
|
||||||
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
if policy_error is not None:
|
||||||
"modules": [],
|
return policy_error
|
||||||
"channel": None,
|
return _valid_catalog_result(catalog_source, state)
|
||||||
"sequence": None,
|
|
||||||
"generated_at": None,
|
|
||||||
"not_before": None,
|
def _catalog_validation_state(
|
||||||
"expires_at": None,
|
source: Path | str | None,
|
||||||
"signed": False,
|
*,
|
||||||
"trusted": False,
|
trusted_keys: dict[str, str],
|
||||||
"key_id": None,
|
) -> _CatalogValidationState:
|
||||||
"warnings": [],
|
payload, read_state = _read_catalog_payload_with_metadata(source)
|
||||||
"error": str(exc),
|
modules = _normalize_catalog_modules(payload)
|
||||||
}
|
channel = _catalog_channel(payload)
|
||||||
if signature_state.get("fatal"):
|
sequence = _catalog_sequence(payload)
|
||||||
return {
|
return _CatalogValidationState(
|
||||||
"valid": False,
|
modules=modules,
|
||||||
"configured": catalog_source is not None,
|
channel=channel,
|
||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
sequence=sequence,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
generated_at=_catalog_optional_text(payload, "generated_at"),
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
not_before=_catalog_optional_text(payload, "not_before"),
|
||||||
"cache_used": bool(read_state.get("cache_used")),
|
expires_at=_catalog_optional_text(payload, "expires_at"),
|
||||||
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
signature_state=_catalog_signature_state(payload, trusted_keys=trusted_keys),
|
||||||
"modules": [],
|
freshness=_catalog_freshness_state(payload),
|
||||||
"channel": channel,
|
replay=_catalog_replay_state(channel=channel, sequence=sequence),
|
||||||
"sequence": sequence,
|
read_state=read_state,
|
||||||
"generated_at": generated_at,
|
)
|
||||||
"not_before": not_before,
|
|
||||||
"expires_at": expires_at,
|
|
||||||
"signed": signature_state["signed"],
|
def _catalog_policy_error(
|
||||||
"trusted": signature_state["trusted"],
|
source: Path | str | None,
|
||||||
"key_id": signature_state["key_id"],
|
state: _CatalogValidationState,
|
||||||
"warnings": [],
|
*,
|
||||||
"error": str(signature_state["error"] or "Module package catalog signature is invalid."),
|
require_trusted: bool,
|
||||||
}
|
approved_channels: tuple[str, ...],
|
||||||
if effective_approved_channels and channel not in effective_approved_channels:
|
) -> dict[str, object] | None:
|
||||||
return {
|
if state.signature_state.get("fatal"):
|
||||||
"valid": False,
|
return _invalid_catalog_state_result(source, state, error=str(state.signature_state["error"] or "Module package catalog signature is invalid."))
|
||||||
"configured": catalog_source is not None,
|
if approved_channels and state.channel not in approved_channels:
|
||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
return _invalid_catalog_state_result(
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
source,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
state,
|
||||||
"cache_used": bool(read_state.get("cache_used")),
|
error=f"Module package catalog channel {state.channel!r} is not approved. Approved channels: {', '.join(approved_channels)}.",
|
||||||
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
|
||||||
"modules": [],
|
|
||||||
"channel": channel,
|
|
||||||
"sequence": sequence,
|
|
||||||
"generated_at": generated_at,
|
|
||||||
"not_before": not_before,
|
|
||||||
"expires_at": expires_at,
|
|
||||||
"signed": signature_state["signed"],
|
|
||||||
"trusted": signature_state["trusted"],
|
|
||||||
"key_id": signature_state["key_id"],
|
|
||||||
"warnings": [],
|
|
||||||
"error": f"Module package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.",
|
|
||||||
}
|
|
||||||
if effective_require_trusted and not signature_state["trusted"]:
|
|
||||||
return {
|
|
||||||
"valid": False,
|
|
||||||
"configured": catalog_source is not None,
|
|
||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
|
||||||
"cache_used": bool(read_state.get("cache_used")),
|
|
||||||
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
|
||||||
"modules": [],
|
|
||||||
"channel": channel,
|
|
||||||
"sequence": sequence,
|
|
||||||
"generated_at": generated_at,
|
|
||||||
"not_before": not_before,
|
|
||||||
"expires_at": expires_at,
|
|
||||||
"signed": signature_state["signed"],
|
|
||||||
"trusted": False,
|
|
||||||
"key_id": signature_state["key_id"],
|
|
||||||
"warnings": [],
|
|
||||||
"error": str(signature_state["error"] or "Module package catalog must be signed by a trusted key."),
|
|
||||||
}
|
|
||||||
if not freshness["valid"]:
|
|
||||||
return _invalid_catalog_result(
|
|
||||||
catalog_source,
|
|
||||||
modules=(),
|
|
||||||
channel=channel,
|
|
||||||
sequence=sequence,
|
|
||||||
generated_at=generated_at,
|
|
||||||
not_before=not_before,
|
|
||||||
expires_at=expires_at,
|
|
||||||
signature_state=signature_state,
|
|
||||||
read_state=read_state,
|
|
||||||
error=str(freshness["error"]),
|
|
||||||
)
|
)
|
||||||
if not replay["valid"]:
|
if require_trusted and not state.signature_state["trusted"]:
|
||||||
return _invalid_catalog_result(
|
return _invalid_catalog_state_result(source, state, error=str(state.signature_state["error"] or "Module package catalog must be signed by a trusted key."))
|
||||||
catalog_source,
|
if not state.freshness["valid"]:
|
||||||
modules=(),
|
return _invalid_catalog_state_result(source, state, error=str(state.freshness["error"]))
|
||||||
channel=channel,
|
if not state.replay["valid"]:
|
||||||
sequence=sequence,
|
return _invalid_catalog_state_result(source, state, error=str(state.replay["error"]))
|
||||||
generated_at=generated_at,
|
return None
|
||||||
not_before=not_before,
|
|
||||||
expires_at=expires_at,
|
|
||||||
signature_state=signature_state,
|
def _catalog_error_result(source: Path | str | None, *, error: str) -> dict[str, object]:
|
||||||
read_state=read_state,
|
return _invalid_catalog_result(
|
||||||
error=str(replay["error"]),
|
source,
|
||||||
)
|
modules=(),
|
||||||
warnings.extend(str(item) for item in freshness.get("warnings", ()) if item)
|
channel=None,
|
||||||
warnings.extend(str(item) for item in replay.get("warnings", ()) if item)
|
sequence=None,
|
||||||
warnings.extend(_catalog_interface_warnings(modules))
|
generated_at=None,
|
||||||
if not signature_state["signed"]:
|
not_before=None,
|
||||||
warnings.append("Catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
expires_at=None,
|
||||||
elif not signature_state["trusted"]:
|
signature_state=_unsigned_catalog_signature_state(),
|
||||||
warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
read_state=_default_catalog_read_state(),
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _invalid_catalog_state_result(source: Path | str | None, state: _CatalogValidationState, *, error: str) -> dict[str, object]:
|
||||||
|
return _invalid_catalog_result(
|
||||||
|
source,
|
||||||
|
modules=(),
|
||||||
|
channel=state.channel,
|
||||||
|
sequence=state.sequence,
|
||||||
|
generated_at=state.generated_at,
|
||||||
|
not_before=state.not_before,
|
||||||
|
expires_at=state.expires_at,
|
||||||
|
signature_state=state.signature_state,
|
||||||
|
read_state=state.read_state,
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_catalog_result(source: Path | str | None, state: _CatalogValidationState) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
"valid": True,
|
"valid": True,
|
||||||
"configured": catalog_source is not None and _catalog_source_exists(catalog_source),
|
"configured": source is not None and _catalog_source_exists(source),
|
||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(source) if source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(source) if source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(source),
|
||||||
"cache_used": bool(read_state.get("cache_used")),
|
"cache_used": bool(state.read_state.get("cache_used")),
|
||||||
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
"cache_path": state.read_state.get("cache_path") if isinstance(state.read_state.get("cache_path"), str) else None,
|
||||||
"modules": list(modules),
|
"modules": list(state.modules),
|
||||||
"channel": channel,
|
"channel": state.channel,
|
||||||
"sequence": sequence,
|
"sequence": state.sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": state.generated_at,
|
||||||
"not_before": not_before,
|
"not_before": state.not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": state.expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": state.signature_state["signed"],
|
||||||
"trusted": signature_state["trusted"],
|
"trusted": state.signature_state["trusted"],
|
||||||
"key_id": signature_state["key_id"],
|
"key_id": state.signature_state["key_id"],
|
||||||
"warnings": warnings,
|
"warnings": _catalog_validation_warnings(state),
|
||||||
"error": None,
|
"error": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_validation_warnings(state: _CatalogValidationState) -> list[str]:
|
||||||
|
warnings: list[str] = []
|
||||||
|
warnings.extend(str(item) for item in state.freshness.get("warnings", ()) if item)
|
||||||
|
warnings.extend(str(item) for item in state.replay.get("warnings", ()) if item)
|
||||||
|
warnings.extend(_catalog_interface_warnings(state.modules))
|
||||||
|
if not state.signature_state["signed"]:
|
||||||
|
warnings.append("Catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
||||||
|
elif not state.signature_state["trusted"]:
|
||||||
|
warnings.append(str(state.signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
||||||
|
return warnings
|
||||||
|
|
||||||
|
|
||||||
|
def _default_catalog_read_state() -> dict[str, object]:
|
||||||
|
cache_path = _configured_catalog_cache_path()
|
||||||
|
return {"cache_used": False, "cache_path": str(cache_path) if cache_path is not None else None}
|
||||||
|
|
||||||
|
|
||||||
|
def _unsigned_catalog_signature_state() -> dict[str, object]:
|
||||||
|
return {"signed": False, "trusted": False, "key_id": None}
|
||||||
|
|
||||||
|
|
||||||
def sign_module_package_catalog(
|
def sign_module_package_catalog(
|
||||||
*,
|
*,
|
||||||
path: Path,
|
path: Path,
|
||||||
@@ -343,17 +329,14 @@ def _configured_trusted_keys_cache_path() -> Path | None:
|
|||||||
|
|
||||||
|
|
||||||
def _read_trusted_keys_url(url: str) -> str:
|
def _read_trusted_keys_url(url: str) -> str:
|
||||||
if not _is_http_url(url):
|
|
||||||
raise ValueError("Trusted catalog key URL must use http:// or https://.")
|
|
||||||
cache_path = _configured_trusted_keys_cache_path()
|
cache_path = _configured_trusted_keys_cache_path()
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=15) as response:
|
body = fetch_http_text(url, timeout=15, label="Trusted catalog key URL")
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return body
|
return body
|
||||||
except (OSError, urllib.error.URLError):
|
except OSError:
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
return cache_path.read_text(encoding="utf-8")
|
return cache_path.read_text(encoding="utf-8")
|
||||||
raise
|
raise
|
||||||
@@ -414,13 +397,12 @@ def _read_catalog_url(url: str) -> str:
|
|||||||
def _read_catalog_url_with_metadata(url: str) -> tuple[str, bool]:
|
def _read_catalog_url_with_metadata(url: str) -> tuple[str, bool]:
|
||||||
cache_path = _configured_catalog_cache_path()
|
cache_path = _configured_catalog_cache_path()
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=15) as response:
|
body = fetch_http_text(url, timeout=15, label="Module package catalog URL")
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return body, False
|
return body, False
|
||||||
except (OSError, urllib.error.URLError):
|
except OSError:
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
return cache_path.read_text(encoding="utf-8"), True
|
return cache_path.read_text(encoding="utf-8"), True
|
||||||
raise
|
raise
|
||||||
@@ -598,7 +580,7 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
|||||||
raise ValueError("Module package catalog entries must be objects.")
|
raise ValueError("Module package catalog entries must be objects.")
|
||||||
module_id = _required_str(value, "module_id")
|
module_id = _required_str(value, "module_id")
|
||||||
action = str(value.get("action") or "install")
|
action = str(value.get("action") or "install")
|
||||||
if action not in {"install", "uninstall"}:
|
if action not in {"install", "update", "uninstall"}:
|
||||||
raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}")
|
raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}")
|
||||||
item = {
|
item = {
|
||||||
"module_id": module_id,
|
"module_id": module_id,
|
||||||
@@ -606,6 +588,21 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
|||||||
"description": _optional_str(value, "description"),
|
"description": _optional_str(value, "description"),
|
||||||
"version": _optional_str(value, "version"),
|
"version": _optional_str(value, "version"),
|
||||||
"action": action,
|
"action": action,
|
||||||
|
"dependencies": _string_list(value.get("dependencies")),
|
||||||
|
"optional_dependencies": _string_list(value.get("optional_dependencies")),
|
||||||
|
"migration_safety": _catalog_migration_safety(value.get("migration_safety"), module_id=module_id),
|
||||||
|
"migration_notes": _optional_str(value, "migration_notes"),
|
||||||
|
"migration_after": _string_list(value.get("migration_after")),
|
||||||
|
"migration_before": _string_list(value.get("migration_before")),
|
||||||
|
"migration_tasks": _normalize_catalog_migration_tasks(value.get("migration_tasks"), module_id=module_id),
|
||||||
|
"current_version_min": _optional_str(value, "current_version_min"),
|
||||||
|
"current_version_max_exclusive": _optional_str(value, "current_version_max_exclusive"),
|
||||||
|
"bridge_release": _optional_bool(value, "bridge_release"),
|
||||||
|
"bridge_notes": _optional_str(value, "bridge_notes"),
|
||||||
|
"allow_downgrade": _optional_bool(value, "allow_downgrade"),
|
||||||
|
"allow_same_version": _optional_bool(value, "allow_same_version"),
|
||||||
|
"recovery_tested": _optional_bool(value, "recovery_tested"),
|
||||||
|
"recovery_notes": _optional_str(value, "recovery_notes"),
|
||||||
"python_package": _optional_str(value, "python_package"),
|
"python_package": _optional_str(value, "python_package"),
|
||||||
"python_ref": _optional_str(value, "python_ref"),
|
"python_ref": _optional_str(value, "python_ref"),
|
||||||
"webui_package": _optional_str(value, "webui_package"),
|
"webui_package": _optional_str(value, "webui_package"),
|
||||||
@@ -616,12 +613,91 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
|||||||
"notes": _optional_str(value, "notes"),
|
"notes": _optional_str(value, "notes"),
|
||||||
"tags": _string_list(value.get("tags")),
|
"tags": _string_list(value.get("tags")),
|
||||||
}
|
}
|
||||||
|
if not version_range_is_valid(
|
||||||
|
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||||
|
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||||
|
):
|
||||||
|
version_range = format_version_range(
|
||||||
|
version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None,
|
||||||
|
version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None,
|
||||||
|
)
|
||||||
|
raise ValueError(f"Module package catalog entry {module_id!r} has invalid current-version range {version_range!r}.")
|
||||||
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
|
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
|
||||||
if artifact_integrity:
|
if artifact_integrity:
|
||||||
item["artifact_integrity"] = artifact_integrity
|
item["artifact_integrity"] = artifact_integrity
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_migration_safety(value: Any, *, module_id: str) -> str:
|
||||||
|
if value is None:
|
||||||
|
return "automatic"
|
||||||
|
safety = str(value).strip()
|
||||||
|
if safety not in CATALOG_MIGRATION_SAFETY:
|
||||||
|
allowed = ", ".join(CATALOG_MIGRATION_SAFETY)
|
||||||
|
raise ValueError(f"Unsupported migration_safety for {module_id!r}: {safety!r}; expected one of {allowed}.")
|
||||||
|
return safety
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_catalog_migration_tasks(value: Any, *, module_id: str) -> list[dict[str, object]]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ValueError(f"Module package catalog migration_tasks for {module_id!r} must be a list.")
|
||||||
|
normalized: list[dict[str, object]] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for raw in value:
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise ValueError(f"Module package catalog migration_tasks entries for {module_id!r} must be objects.")
|
||||||
|
task_id = _catalog_task_required_str(raw, "task_id", module_id=module_id)
|
||||||
|
if task_id in seen:
|
||||||
|
raise ValueError(f"Module package catalog entry {module_id!r} declares migration task {task_id!r} more than once.")
|
||||||
|
seen.add(task_id)
|
||||||
|
phase = _catalog_task_required_str(raw, "phase", module_id=module_id)
|
||||||
|
if phase not in CATALOG_MIGRATION_TASK_PHASES:
|
||||||
|
allowed = ", ".join(CATALOG_MIGRATION_TASK_PHASES)
|
||||||
|
raise ValueError(f"Unsupported migration task phase for {module_id!r}/{task_id!r}: {phase!r}; expected one of {allowed}.")
|
||||||
|
summary = _catalog_task_required_str(raw, "summary", module_id=module_id)
|
||||||
|
safety = _catalog_migration_safety(raw.get("safety"), module_id=module_id)
|
||||||
|
item: dict[str, object] = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"phase": phase,
|
||||||
|
"summary": summary,
|
||||||
|
"task_version": _optional_str(raw, "task_version") or "1",
|
||||||
|
"safety": safety,
|
||||||
|
"idempotent": _catalog_optional_bool_default(raw, "idempotent", default=True),
|
||||||
|
}
|
||||||
|
timeout_seconds = _catalog_optional_positive_int(raw, "timeout_seconds", module_id=module_id, task_id=task_id)
|
||||||
|
if timeout_seconds is not None:
|
||||||
|
item["timeout_seconds"] = timeout_seconds
|
||||||
|
normalized.append(item)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_task_required_str(value: dict[str, Any], key: str, *, module_id: str) -> str:
|
||||||
|
item = _optional_str(value, key)
|
||||||
|
if not item:
|
||||||
|
raise ValueError(f"Module package catalog migration task for {module_id!r} is missing {key!r}.")
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_optional_bool_default(value: dict[str, Any], key: str, *, default: bool) -> bool:
|
||||||
|
if key not in value:
|
||||||
|
return default
|
||||||
|
return _optional_bool(value, key)
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_optional_positive_int(value: dict[str, Any], key: str, *, module_id: str, task_id: str) -> int | None:
|
||||||
|
if value.get(key) is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
integer = int(value[key])
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} has invalid {key!r}.") from exc
|
||||||
|
if integer <= 0:
|
||||||
|
raise ValueError(f"Module package catalog migration task {module_id!r}/{task_id!r} requires positive {key!r}.")
|
||||||
|
return integer
|
||||||
|
|
||||||
|
|
||||||
def _required_str(value: dict[str, Any], key: str) -> str:
|
def _required_str(value: dict[str, Any], key: str) -> str:
|
||||||
item = _optional_str(value, key)
|
item = _optional_str(value, key)
|
||||||
if not item:
|
if not item:
|
||||||
@@ -834,7 +910,7 @@ def _catalog_source_type(source: Path | str | None) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _is_http_url(value: str) -> bool:
|
def _is_http_url(value: str) -> bool:
|
||||||
return value.startswith(("https://", "http://"))
|
return is_http_url(value)
|
||||||
|
|
||||||
|
|
||||||
def _invalid_catalog_result(
|
def _invalid_catalog_result(
|
||||||
|
|||||||
@@ -13,6 +13,14 @@ SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1"
|
|||||||
|
|
||||||
PermissionLevel = Literal["system", "tenant"]
|
PermissionLevel = Literal["system", "tenant"]
|
||||||
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
||||||
|
MigrationTaskPhase = Literal[
|
||||||
|
"pre_migration_check",
|
||||||
|
"pre_migration_prepare",
|
||||||
|
"post_migration_backfill",
|
||||||
|
"post_migration_verify",
|
||||||
|
]
|
||||||
|
MigrationTaskSafety = Literal["automatic", "requires_review", "forward_only", "destructive"]
|
||||||
|
MigrationTaskStatus = Literal["ok", "warning", "blocked", "skipped"]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -91,11 +99,49 @@ MigrationRetirementExecutor = Callable[[object, str], None]
|
|||||||
MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan]
|
MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ModuleMigrationTaskContext:
|
||||||
|
module_id: str
|
||||||
|
task_id: str
|
||||||
|
phase: MigrationTaskPhase
|
||||||
|
database_url: str
|
||||||
|
target_version: str | None = None
|
||||||
|
session: object | None = None
|
||||||
|
dry_run: bool = False
|
||||||
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ModuleMigrationTaskResult:
|
||||||
|
status: MigrationTaskStatus = "ok"
|
||||||
|
message: str | None = None
|
||||||
|
details: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
ModuleMigrationTaskExecutor = Callable[[ModuleMigrationTaskContext], ModuleMigrationTaskResult | None]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ModuleMigrationTask:
|
||||||
|
task_id: str
|
||||||
|
phase: MigrationTaskPhase
|
||||||
|
summary: str
|
||||||
|
task_version: str = "1"
|
||||||
|
safety: MigrationTaskSafety = "automatic"
|
||||||
|
idempotent: bool = True
|
||||||
|
timeout_seconds: int | None = None
|
||||||
|
executor: ModuleMigrationTaskExecutor | None = None
|
||||||
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class MigrationSpec:
|
class MigrationSpec:
|
||||||
module_id: str
|
module_id: str
|
||||||
metadata: object | None = None
|
metadata: object | None = None
|
||||||
script_location: str | None = None
|
script_location: str | None = None
|
||||||
|
migration_after: tuple[str, ...] = ()
|
||||||
|
migration_before: tuple[str, ...] = ()
|
||||||
|
migration_tasks: tuple[ModuleMigrationTask, ...] = ()
|
||||||
retirement_supported: bool = False
|
retirement_supported: bool = False
|
||||||
retirement_provider: MigrationRetirementProvider | None = None
|
retirement_provider: MigrationRetirementProvider | None = None
|
||||||
retirement_notes: str | None = None
|
retirement_notes: str | None = None
|
||||||
@@ -215,7 +261,28 @@ class ResourceAclProvider(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
|
TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
|
||||||
DeleteVetoProvider = Callable[[object, str, str], None]
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DeleteVetoIssue:
|
||||||
|
severity: Literal["blocker", "warning", "info"]
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
module_id: str | None = None
|
||||||
|
details: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
DeleteVetoProviderResult = DeleteVetoIssue | Iterable[DeleteVetoIssue] | None
|
||||||
|
DeleteVetoProvider = Callable[[object, str, str], DeleteVetoProviderResult]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DeleteVetoProviderRegistration:
|
||||||
|
module_id: str
|
||||||
|
resource_type: str
|
||||||
|
provider: DeleteVetoProvider
|
||||||
|
|
||||||
|
|
||||||
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
||||||
CapabilityFactory = Callable[[ModuleContext], object]
|
CapabilityFactory = Callable[[ModuleContext], object]
|
||||||
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]
|
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]
|
||||||
|
|||||||
64
src/govoplan_core/core/notifications.py
Normal file
64
src/govoplan_core/core/notifications.py
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
CAPABILITY_NOTIFICATIONS_DISPATCH = "notifications.dispatch"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class NotificationDispatchRequest:
|
||||||
|
tenant_id: str
|
||||||
|
source_module: str
|
||||||
|
source_resource_type: str
|
||||||
|
source_resource_id: str | None
|
||||||
|
event_kind: str
|
||||||
|
channel: str = "inbox"
|
||||||
|
recipient: str | None = None
|
||||||
|
recipient_type: str | None = None
|
||||||
|
recipient_id: str | None = None
|
||||||
|
recipient_label: str | None = None
|
||||||
|
subject: str | None = None
|
||||||
|
body_text: str | None = None
|
||||||
|
body_html: str | None = None
|
||||||
|
action_url: str | None = None
|
||||||
|
priority: int = 0
|
||||||
|
not_before_at: datetime | None = None
|
||||||
|
payload: Mapping[str, object] = field(default_factory=dict)
|
||||||
|
metadata: Mapping[str, object] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class NotificationDispatchProvider(Protocol):
|
||||||
|
def enqueue_notification(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
request: NotificationDispatchRequest,
|
||||||
|
*,
|
||||||
|
enqueue_delivery: bool = True,
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def deliver_notification(self, session: object, *, notification_id: str) -> Mapping[str, object]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def deliver_pending(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
def notification_dispatch_provider(registry: object | None) -> NotificationDispatchProvider | None:
|
||||||
|
if registry is None or not hasattr(registry, "has_capability"):
|
||||||
|
return None
|
||||||
|
if not registry.has_capability(CAPABILITY_NOTIFICATIONS_DISPATCH):
|
||||||
|
return None
|
||||||
|
capability = registry.capability(CAPABILITY_NOTIFICATIONS_DISPATCH)
|
||||||
|
return capability if isinstance(capability, NotificationDispatchProvider) else None
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user