chore: consolidate platform split checks
This commit is contained in:
34
.gitea/workflows/dependency-audit.yml
Normal file
34
.gitea/workflows/dependency-audit.yml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
name: Dependency Audit
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
schedule:
|
||||||
|
- cron: "23 3 * * 1"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
dependency-audit:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Install backend dev audit dependencies
|
||||||
|
run: |
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/bin/python -m pip install --upgrade pip
|
||||||
|
.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
.venv/bin/python -m pip install 'pip-audit>=2.9,<3'
|
||||||
|
- name: Install WebUI release dependencies
|
||||||
|
working-directory: webui
|
||||||
|
run: |
|
||||||
|
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); pkg.dependencies=rel.dependencies; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
||||||
|
npm install
|
||||||
|
- name: Run dependency audits
|
||||||
|
run: bash scripts/check-dependency-audits.sh
|
||||||
31
.gitea/workflows/module-matrix.yml
Normal file
31
.gitea/workflows/module-matrix.yml
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
name: Module Matrix
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
module-matrix:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
- name: Install backend release dependencies
|
||||||
|
run: |
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/bin/python -m pip install --upgrade pip
|
||||||
|
.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
- name: Install WebUI release dependencies with test scripts
|
||||||
|
working-directory: webui
|
||||||
|
run: |
|
||||||
|
node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); pkg.dependencies=rel.dependencies; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
|
||||||
|
npm install
|
||||||
|
- name: Run module matrix and contract tests
|
||||||
|
run: bash scripts/check-module-matrix.sh
|
||||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -136,6 +136,18 @@ dist
|
|||||||
.yarn/install-state.gz
|
.yarn/install-state.gz
|
||||||
.pnp.*
|
.pnp.*
|
||||||
|
|
||||||
|
# Local WebUI test/build scratch directories
|
||||||
|
.component-test-build/
|
||||||
|
.module-test-build/
|
||||||
|
.policy-test-build/
|
||||||
|
.template-preview-test-build/
|
||||||
|
.import-test-build/
|
||||||
|
webui/.component-test-build/
|
||||||
|
webui/.module-test-build/
|
||||||
|
webui/.policy-test-build/
|
||||||
|
webui/.template-preview-test-build/
|
||||||
|
webui/.import-test-build/
|
||||||
|
|
||||||
# ---> Python
|
# ---> Python
|
||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
|||||||
41
README.md
41
README.md
@@ -1,6 +1,6 @@
|
|||||||
# govoplan-core
|
# govoplan-core
|
||||||
|
|
||||||
GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, tenant and RBAC infrastructure, governance policy, audit/auth helpers, module discovery, migration registration, and the shared WebUI shell. Feature code is supplied by installed modules.
|
GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, module discovery, migration orchestration, capability contracts, install/uninstall orchestration, and the shared WebUI shell. Platform and feature behavior is supplied by installed modules.
|
||||||
|
|
||||||
## Repository ownership
|
## Repository ownership
|
||||||
|
|
||||||
@@ -9,22 +9,28 @@ Core owns:
|
|||||||
- `govoplan_core.server.app:app`, the FastAPI entry point used by uvicorn
|
- `govoplan_core.server.app:app`, the FastAPI entry point used by uvicorn
|
||||||
- `GovoplanServerConfig`, module discovery, registry validation, and route aggregation
|
- `GovoplanServerConfig`, module discovery, registry validation, and route aggregation
|
||||||
- SQLAlchemy base/session helpers and module migration registration
|
- SQLAlchemy base/session helpers and module migration registration
|
||||||
- tenant/account/session/RBAC/governance/audit models and services
|
- kernel APIs for platform metadata, module lifecycle, health, and development diagnostics
|
||||||
- core API routes for auth, admin, platform metadata, audit, and system health
|
|
||||||
- `@govoplan/core-webui`, including login, CSRF/API helpers, shell layout, generic UI components, IconRail, DataGrid, access boundaries, and module route/nav contracts
|
- `@govoplan/core-webui`, including login, CSRF/API helpers, shell layout, generic UI components, IconRail, DataGrid, access boundaries, and module route/nav contracts
|
||||||
|
|
||||||
Feature modules own their backend routers, models, migrations, permissions, frontend packages, nav items, and route contributions. Core should not import feature pages directly; it imports module manifests and renders their route contributions.
|
Platform and feature modules own their backend routers, models, migrations,
|
||||||
|
permissions, frontend packages, nav items, and route contributions. Access,
|
||||||
|
tenancy, policy, audit, and admin behavior live in their owning platform
|
||||||
|
modules. Core should not import feature pages directly; it imports module
|
||||||
|
manifests and renders their route contributions.
|
||||||
|
|
||||||
## Governance docs
|
## Governance docs
|
||||||
|
|
||||||
Canonical policy documents live in `docs/`:
|
Canonical policy documents live in `docs/`:
|
||||||
|
|
||||||
- [RBAC_MANIFEST.md](docs/RBAC_MANIFEST.md)
|
- [DOCUMENTATION_MAP.md](docs/DOCUMENTATION_MAP.md)
|
||||||
- [SYSTEM_GOVERNANCE_MANIFEST.md](docs/SYSTEM_GOVERNANCE_MANIFEST.md)
|
- [ACCESS_RBAC_MODEL.md](docs/ACCESS_RBAC_MODEL.md)
|
||||||
|
- [GOVERNANCE_MODEL.md](docs/GOVERNANCE_MODEL.md)
|
||||||
- [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md)
|
- [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md)
|
||||||
|
- [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md)
|
||||||
- [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md)
|
- [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md)
|
||||||
|
|
||||||
Modules may define module-specific permissions and policy behavior, but the platform-level permission model and governance hierarchy belong here.
|
Modules define module-specific permissions and policy behavior. Shared DTOs and
|
||||||
|
composition rules live in core only where they are stable kernel contracts.
|
||||||
|
|
||||||
## Backend development
|
## Backend development
|
||||||
|
|
||||||
@@ -35,7 +41,7 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
./.venv/bin/python -m pip install -r requirements-dev.txt
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,access,admin,policy,audit,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation.
|
Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
@@ -55,13 +61,24 @@ ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \
|
|||||||
|
|
||||||
The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`.
|
The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`.
|
||||||
|
|
||||||
The default development SQLite database lives at `runtime/multimailer-dev.db`, alongside other local runtime state.
|
The default development database is PostgreSQL at `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev`. Store the password in `~/.pgpass`. To force the disposable SQLite fallback, run with `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite`; that database lives below `runtime/`.
|
||||||
|
|
||||||
Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running.
|
Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running.
|
||||||
|
|
||||||
If the configured local SQLite database is missing or empty, `govoplan_core.devserver` enables the development bootstrap before loading settings. This creates the schema and the default development login on startup. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
To run the production-like local profile with PostgreSQL, Redis, a Celery
|
||||||
|
worker, explicit module configuration, and persistent local file storage:
|
||||||
|
|
||||||
To verify the effective runtime paths and missing-SQLite bootstrap without starting uvicorn, run the smoke mode:
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
See [dev/production-like/README.md](dev/production-like/README.md) for ports,
|
||||||
|
environment overrides, and cleanup commands.
|
||||||
|
|
||||||
|
`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead.
|
||||||
|
|
||||||
|
To verify the effective runtime paths and bootstrap behavior without starting uvicorn, run the smoke mode:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
@@ -72,6 +89,8 @@ The smoke mode prints the effective config, runtime root, database URL, modules,
|
|||||||
|
|
||||||
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md).
|
||||||
|
|
||||||
|
For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md).
|
||||||
|
|
||||||
## WebUI development
|
## WebUI development
|
||||||
|
|
||||||
Install and run from the core WebUI host:
|
Install and run from the core WebUI host:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from sqlalchemy import engine_from_config, pool
|
|||||||
|
|
||||||
from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata
|
from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata
|
||||||
from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata
|
from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata
|
||||||
|
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||||
from govoplan_core.core.migrations import migration_metadata_plan
|
from govoplan_core.core.migrations import migration_metadata_plan
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_core.server.default_config import get_server_config
|
from govoplan_core.server.default_config import get_server_config
|
||||||
|
|||||||
@@ -94,10 +94,10 @@ def _scrub_policy_column(table_name: str) -> None:
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
inspector = sa.inspect(op.get_bind())
|
inspector = sa.inspect(op.get_bind())
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("system_settings", "tenants"):
|
for table_name in ("core_system_settings", "tenancy_tenants"):
|
||||||
if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}:
|
if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}:
|
||||||
_scrub_settings_table(table_name)
|
_scrub_settings_table(table_name)
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}:
|
if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}:
|
||||||
_scrub_policy_column(table_name)
|
_scrub_policy_column(table_name)
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ def upgrade() -> None:
|
|||||||
batch_op.add_column(sa.Column("locked_by_user_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("locked_by_user_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
op.f("fk_campaign_versions_locked_by_user_id_users"),
|
op.f("fk_campaign_versions_locked_by_user_id_users"),
|
||||||
"users",
|
"access_users",
|
||||||
["locked_by_user_id"],
|
["locked_by_user_id"],
|
||||||
["id"],
|
["id"],
|
||||||
ondelete="SET NULL",
|
ondelete="SET NULL",
|
||||||
|
|||||||
@@ -17,67 +17,67 @@ depends_on = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.add_column(sa.Column("auth_provider", sa.String(length=50), nullable=False, server_default="local"))
|
batch_op.add_column(sa.Column("auth_provider", sa.String(length=50), nullable=False, server_default="local"))
|
||||||
batch_op.add_column(sa.Column("password_hash", sa.String(length=500), nullable=True))
|
batch_op.add_column(sa.Column("password_hash", sa.String(length=500), nullable=True))
|
||||||
batch_op.add_column(sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
batch_op.add_column(sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"user_group_memberships",
|
"access_user_group_memberships",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("group_id", sa.String(length=36), nullable=False),
|
sa.Column("group_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"),
|
sa.UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_user_group_memberships_tenant_id"), "user_group_memberships", ["tenant_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_tenant_id"), "access_user_group_memberships", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_user_group_memberships_user_id"), "user_group_memberships", ["user_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_user_id"), "access_user_group_memberships", ["user_id"])
|
||||||
op.create_index(op.f("ix_user_group_memberships_group_id"), "user_group_memberships", ["group_id"])
|
op.create_index(op.f("ix_access_user_group_memberships_group_id"), "access_user_group_memberships", ["group_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"user_role_assignments",
|
"access_user_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"),
|
sa.UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_user_role_assignments_tenant_id"), "user_role_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_tenant_id"), "access_user_role_assignments", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_user_role_assignments_user_id"), "user_role_assignments", ["user_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_user_id"), "access_user_role_assignments", ["user_id"])
|
||||||
op.create_index(op.f("ix_user_role_assignments_role_id"), "user_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_user_role_assignments_role_id"), "access_user_role_assignments", ["role_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"group_role_assignments",
|
"access_group_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("group_id", sa.String(length=36), nullable=False),
|
sa.Column("group_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"),
|
sa.UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_group_role_assignments_tenant_id"), "group_role_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_tenant_id"), "access_group_role_assignments", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_group_role_assignments_group_id"), "group_role_assignments", ["group_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_group_id"), "access_group_role_assignments", ["group_id"])
|
||||||
op.create_index(op.f("ix_group_role_assignments_role_id"), "group_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_group_role_assignments_role_id"), "access_group_role_assignments", ["role_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"auth_sessions",
|
"access_auth_sessions",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("user_id", sa.String(length=36), nullable=False),
|
sa.Column("user_id", sa.String(length=36), nullable=False),
|
||||||
@@ -89,42 +89,42 @@ def upgrade() -> None:
|
|||||||
sa.Column("ip_address", sa.String(length=100), nullable=True),
|
sa.Column("ip_address", sa.String(length=100), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("token_hash"),
|
sa.UniqueConstraint("token_hash"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_auth_sessions_tenant_id"), "auth_sessions", ["tenant_id"])
|
op.create_index(op.f("ix_access_auth_sessions_tenant_id"), "access_auth_sessions", ["tenant_id"])
|
||||||
op.create_index(op.f("ix_auth_sessions_user_id"), "auth_sessions", ["user_id"])
|
op.create_index(op.f("ix_access_auth_sessions_user_id"), "access_auth_sessions", ["user_id"])
|
||||||
op.create_index(op.f("ix_auth_sessions_token_hash"), "auth_sessions", ["token_hash"])
|
op.create_index(op.f("ix_access_auth_sessions_token_hash"), "access_auth_sessions", ["token_hash"])
|
||||||
op.create_index(op.f("ix_auth_sessions_expires_at"), "auth_sessions", ["expires_at"])
|
op.create_index(op.f("ix_access_auth_sessions_expires_at"), "access_auth_sessions", ["expires_at"])
|
||||||
op.create_index(op.f("ix_auth_sessions_revoked_at"), "auth_sessions", ["revoked_at"])
|
op.create_index(op.f("ix_access_auth_sessions_revoked_at"), "access_auth_sessions", ["revoked_at"])
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index(op.f("ix_auth_sessions_revoked_at"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_revoked_at"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_expires_at"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_expires_at"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_token_hash"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_token_hash"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_user_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_user_id"), table_name="access_auth_sessions")
|
||||||
op.drop_index(op.f("ix_auth_sessions_tenant_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_tenant_id"), table_name="access_auth_sessions")
|
||||||
op.drop_table("auth_sessions")
|
op.drop_table("access_auth_sessions")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_group_role_assignments_role_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_role_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_index(op.f("ix_group_role_assignments_group_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_group_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_index(op.f("ix_group_role_assignments_tenant_id"), table_name="group_role_assignments")
|
op.drop_index(op.f("ix_access_group_role_assignments_tenant_id"), table_name="access_group_role_assignments")
|
||||||
op.drop_table("group_role_assignments")
|
op.drop_table("access_group_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_user_role_assignments_role_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_role_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_index(op.f("ix_user_role_assignments_user_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_user_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_index(op.f("ix_user_role_assignments_tenant_id"), table_name="user_role_assignments")
|
op.drop_index(op.f("ix_access_user_role_assignments_tenant_id"), table_name="access_user_role_assignments")
|
||||||
op.drop_table("user_role_assignments")
|
op.drop_table("access_user_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_user_group_memberships_group_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_group_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_index(op.f("ix_user_group_memberships_user_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_user_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_index(op.f("ix_user_group_memberships_tenant_id"), table_name="user_group_memberships")
|
op.drop_index(op.f("ix_access_user_group_memberships_tenant_id"), table_name="access_user_group_memberships")
|
||||||
op.drop_table("user_group_memberships")
|
op.drop_table("access_user_group_memberships")
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.drop_column("last_login_at")
|
batch_op.drop_column("last_login_at")
|
||||||
batch_op.drop_column("password_hash")
|
batch_op.drop_column("password_hash")
|
||||||
batch_op.drop_column("auth_provider")
|
batch_op.drop_column("auth_provider")
|
||||||
|
|||||||
78
alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py
Normal file
78
alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
"""namespace platform-owned tables
|
||||||
|
|
||||||
|
Revision ID: 2e3f4a5b6c7d
|
||||||
|
Revises: 1b2c3d4e5f70
|
||||||
|
Create Date: 2026-07-09 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "2e3f4a5b6c7d"
|
||||||
|
down_revision = "1b2c3d4e5f70"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
_TABLE_RENAMES = (
|
||||||
|
("tenants", "tenancy_tenants"),
|
||||||
|
("accounts", "access_accounts"),
|
||||||
|
("users", "access_users"),
|
||||||
|
("groups", "access_groups"),
|
||||||
|
("roles", "access_roles"),
|
||||||
|
("system_role_assignments", "access_system_role_assignments"),
|
||||||
|
("user_group_memberships", "access_user_group_memberships"),
|
||||||
|
("user_role_assignments", "access_user_role_assignments"),
|
||||||
|
("group_role_assignments", "access_group_role_assignments"),
|
||||||
|
("api_keys", "access_api_keys"),
|
||||||
|
("auth_sessions", "access_auth_sessions"),
|
||||||
|
("system_settings", "core_system_settings"),
|
||||||
|
("governance_templates", "admin_governance_templates"),
|
||||||
|
("governance_template_assignments", "admin_governance_template_assignments"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _row_count(bind: sa.Connection, table_name: str) -> int:
|
||||||
|
return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
|
old_tables_to_drop: set[str] = set()
|
||||||
|
new_tables_to_drop: set[str] = set()
|
||||||
|
for old_name, new_name in renames:
|
||||||
|
if old_name not in tables or new_name not in tables:
|
||||||
|
continue
|
||||||
|
if _row_count(bind, old_name) == 0:
|
||||||
|
old_tables_to_drop.add(old_name)
|
||||||
|
elif _row_count(bind, new_name) == 0:
|
||||||
|
new_tables_to_drop.add(new_name)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Cannot rename non-empty {old_name} over non-empty {new_name}")
|
||||||
|
|
||||||
|
for old_name, _new_name in reversed(renames):
|
||||||
|
if old_name in old_tables_to_drop:
|
||||||
|
op.drop_table(old_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
for _old_name, new_name in reversed(renames):
|
||||||
|
if new_name in new_tables_to_drop:
|
||||||
|
op.drop_table(new_name)
|
||||||
|
tables.remove(new_name)
|
||||||
|
|
||||||
|
for old_name, new_name in renames:
|
||||||
|
if old_name not in tables:
|
||||||
|
continue
|
||||||
|
op.rename_table(old_name, new_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
tables.add(new_name)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
_rename_tables(_TABLE_RENAMES)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
_rename_tables(tuple((new_name, old_name) for old_name, new_name in reversed(_TABLE_RENAMES)))
|
||||||
@@ -32,7 +32,7 @@ def upgrade() -> None:
|
|||||||
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),
|
sa.UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"),
|
||||||
)
|
)
|
||||||
@@ -55,10 +55,10 @@ def upgrade() -> None:
|
|||||||
sa.Column("metadata", sa.JSON(), nullable=True),
|
sa.Column("metadata", sa.JSON(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "current_version_id", "display_path", "filename", "created_by_user_id", "deleted_at"]:
|
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "current_version_id", "display_path", "filename", "created_by_user_id", "deleted_at"]:
|
||||||
@@ -80,9 +80,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),
|
sa.UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"),
|
||||||
)
|
)
|
||||||
@@ -101,9 +101,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),
|
sa.UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"),
|
||||||
)
|
)
|
||||||
@@ -136,7 +136,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["file_blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_blob_id"], ["file_blobs.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["file_version_id"], ["file_versions.id"], ondelete="RESTRICT"),
|
sa.ForeignKeyConstraint(["file_version_id"], ["file_versions.id"], ondelete="RESTRICT"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),
|
sa.UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"),
|
||||||
)
|
)
|
||||||
|
|||||||
111
alembic/versions/3f4a5b6c7d8e_core_change_sequence.py
Normal file
111
alembic/versions/3f4a5b6c7d8e_core_change_sequence.py
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
"""add core change sequence
|
||||||
|
|
||||||
|
Revision ID: 3f4a5b6c7d8e
|
||||||
|
Revises: 2e3f4a5b6c7d
|
||||||
|
Create Date: 2026-07-09 00:00:00.000000
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "3f4a5b6c7d8e"
|
||||||
|
down_revision = "2e3f4a5b6c7d"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence" not in inspector.get_table_names():
|
||||||
|
op.create_table(
|
||||||
|
"core_change_sequence",
|
||||||
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=True),
|
||||||
|
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("collection", sa.String(length=150), nullable=False),
|
||||||
|
sa.Column("resource_type", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("resource_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("operation", sa.String(length=30), nullable=False),
|
||||||
|
sa.Column("actor_type", sa.String(length=30), nullable=True),
|
||||||
|
sa.Column("actor_id", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("payload", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence")),
|
||||||
|
)
|
||||||
|
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")}
|
||||||
|
for column in (
|
||||||
|
"tenant_id",
|
||||||
|
"module_id",
|
||||||
|
"collection",
|
||||||
|
"resource_type",
|
||||||
|
"resource_id",
|
||||||
|
"operation",
|
||||||
|
"actor_type",
|
||||||
|
"actor_id",
|
||||||
|
"created_at",
|
||||||
|
):
|
||||||
|
name = op.f(f"ix_core_change_sequence_{column}")
|
||||||
|
if name not in indexes:
|
||||||
|
op.create_index(name, "core_change_sequence", [column], unique=False)
|
||||||
|
for name, columns in (
|
||||||
|
("ix_core_change_sequence_tenant_module_id", ["tenant_id", "module_id", "id"]),
|
||||||
|
("ix_core_change_sequence_collection_id", ["collection", "id"]),
|
||||||
|
("ix_core_change_sequence_resource", ["module_id", "resource_type", "resource_id"]),
|
||||||
|
):
|
||||||
|
if name not in indexes:
|
||||||
|
op.create_index(name, "core_change_sequence", columns, unique=False)
|
||||||
|
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence_retention_floor" not in inspector.get_table_names():
|
||||||
|
op.create_table(
|
||||||
|
"core_change_sequence_retention_floor",
|
||||||
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("tenant_key", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("module_id", sa.String(length=100), nullable=False),
|
||||||
|
sa.Column("collection", sa.String(length=150), nullable=False),
|
||||||
|
sa.Column("min_valid_sequence", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence_retention_floor")),
|
||||||
|
sa.UniqueConstraint("tenant_key", "module_id", "collection", name="uq_core_change_sequence_retention_scope"),
|
||||||
|
)
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")}
|
||||||
|
if "ix_core_change_sequence_retention_scope" not in indexes:
|
||||||
|
op.create_index(
|
||||||
|
"ix_core_change_sequence_retention_scope",
|
||||||
|
"core_change_sequence_retention_floor",
|
||||||
|
["tenant_key", "module_id", "collection"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if "core_change_sequence_retention_floor" in inspector.get_table_names():
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")}
|
||||||
|
if "ix_core_change_sequence_retention_scope" in indexes:
|
||||||
|
op.drop_index("ix_core_change_sequence_retention_scope", table_name="core_change_sequence_retention_floor")
|
||||||
|
op.drop_table("core_change_sequence_retention_floor")
|
||||||
|
if "core_change_sequence" not in inspector.get_table_names():
|
||||||
|
return
|
||||||
|
indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")}
|
||||||
|
for name in (
|
||||||
|
"ix_core_change_sequence_resource",
|
||||||
|
"ix_core_change_sequence_collection_id",
|
||||||
|
"ix_core_change_sequence_tenant_module_id",
|
||||||
|
op.f("ix_core_change_sequence_created_at"),
|
||||||
|
op.f("ix_core_change_sequence_actor_id"),
|
||||||
|
op.f("ix_core_change_sequence_actor_type"),
|
||||||
|
op.f("ix_core_change_sequence_operation"),
|
||||||
|
op.f("ix_core_change_sequence_resource_id"),
|
||||||
|
op.f("ix_core_change_sequence_resource_type"),
|
||||||
|
op.f("ix_core_change_sequence_collection"),
|
||||||
|
op.f("ix_core_change_sequence_module_id"),
|
||||||
|
op.f("ix_core_change_sequence_tenant_id"),
|
||||||
|
):
|
||||||
|
if name in indexes:
|
||||||
|
op.drop_index(name, table_name="core_change_sequence")
|
||||||
|
op.drop_table("core_change_sequence")
|
||||||
@@ -31,10 +31,10 @@ def upgrade() -> None:
|
|||||||
sa.Column("metadata", sa.JSON(), nullable=True),
|
sa.Column("metadata", sa.JSON(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "path", "created_by_user_id", "deleted_at"]:
|
for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "path", "created_by_user_id", "deleted_at"]:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ def upgrade() -> None:
|
|||||||
batch_op.add_column(sa.Column("user_locked_by_user_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("user_locked_by_user_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_campaign_versions_user_locked_by_user_id_users",
|
"fk_campaign_versions_user_locked_by_user_id_users",
|
||||||
"users",
|
"access_users",
|
||||||
["user_locked_by_user_id"],
|
["user_locked_by_user_id"],
|
||||||
["id"],
|
["id"],
|
||||||
ondelete="SET NULL",
|
ondelete="SET NULL",
|
||||||
|
|||||||
@@ -62,25 +62,25 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
user_columns = {column["name"] for column in inspector.get_columns("users")}
|
user_columns = {column["name"] for column in inspector.get_columns("access_users")}
|
||||||
# Base.metadata.create_all() from a newer application can create brand-new
|
# Base.metadata.create_all() from a newer application can create brand-new
|
||||||
# tables while leaving existing tables unaltered. Repair that known drift by
|
# tables while leaving existing tables unaltered. Repair that known drift by
|
||||||
# removing only empty, unreferenced administration tables before applying the
|
# removing only empty, unreferenced administration tables before applying the
|
||||||
# real migration. A non-empty table is never guessed at or discarded.
|
# real migration. A non-empty table is never guessed at or discarded.
|
||||||
if "account_id" not in user_columns and "system_role_assignments" in tables:
|
if "account_id" not in user_columns and "access_system_role_assignments" in tables:
|
||||||
count = bind.execute(sa.text("SELECT COUNT(*) FROM system_role_assignments")).scalar_one()
|
count = bind.execute(sa.text("SELECT COUNT(*) FROM access_system_role_assignments")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
raise RuntimeError("Cannot reconcile non-empty create_all system_role_assignments table")
|
raise RuntimeError("Cannot reconcile non-empty create_all system_role_assignments table")
|
||||||
op.drop_table("system_role_assignments")
|
op.drop_table("access_system_role_assignments")
|
||||||
tables.remove("system_role_assignments")
|
tables.remove("access_system_role_assignments")
|
||||||
if "account_id" not in user_columns and "accounts" in tables:
|
if "account_id" not in user_columns and "access_accounts" in tables:
|
||||||
count = bind.execute(sa.text("SELECT COUNT(*) FROM accounts")).scalar_one()
|
count = bind.execute(sa.text("SELECT COUNT(*) FROM access_accounts")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
raise RuntimeError("Cannot reconcile non-empty create_all accounts table")
|
raise RuntimeError("Cannot reconcile non-empty create_all accounts table")
|
||||||
op.drop_table("accounts")
|
op.drop_table("access_accounts")
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"accounts",
|
"access_accounts",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("email", sa.String(length=320), nullable=False),
|
sa.Column("email", sa.String(length=320), nullable=False),
|
||||||
sa.Column("normalized_email", sa.String(length=320), nullable=False),
|
sa.Column("normalized_email", sa.String(length=320), nullable=False),
|
||||||
@@ -95,29 +95,29 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("normalized_email", name="uq_accounts_normalized_email"),
|
sa.UniqueConstraint("normalized_email", name="uq_accounts_normalized_email"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_accounts_normalized_email"), "accounts", ["normalized_email"])
|
op.create_index(op.f("ix_access_accounts_normalized_email"), "access_accounts", ["normalized_email"])
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"))
|
batch_op.add_column(sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"))
|
||||||
batch_op.add_column(sa.Column("settings", sa.JSON(), nullable=False, server_default="{}"))
|
batch_op.add_column(sa.Column("settings", sa.JSON(), nullable=False, server_default="{}"))
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()))
|
batch_op.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
batch_op.add_column(sa.Column("description", sa.Text(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.add_column(sa.Column("is_assignable", sa.Boolean(), nullable=False, server_default=sa.true()))
|
batch_op.add_column(sa.Column("is_assignable", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True))
|
||||||
|
|
||||||
users = bind.execute(sa.text(
|
users = bind.execute(sa.text(
|
||||||
"SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM users ORDER BY created_at, id"
|
"SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM access_users ORDER BY created_at, id"
|
||||||
)).mappings().all()
|
)).mappings().all()
|
||||||
|
|
||||||
accounts_by_email: dict[str, str] = {}
|
accounts_by_email: dict[str, str] = {}
|
||||||
@@ -155,7 +155,7 @@ def upgrade() -> None:
|
|||||||
first_system_owner_account_id = account_id
|
first_system_owner_account_id = account_id
|
||||||
|
|
||||||
accounts_table = sa.table(
|
accounts_table = sa.table(
|
||||||
"accounts",
|
"access_accounts",
|
||||||
sa.column("id", sa.String), sa.column("email", sa.String), sa.column("normalized_email", sa.String),
|
sa.column("id", sa.String), sa.column("email", sa.String), sa.column("normalized_email", sa.String),
|
||||||
sa.column("display_name", sa.String), sa.column("is_active", sa.Boolean), sa.column("auth_provider", sa.String),
|
sa.column("display_name", sa.String), sa.column("is_active", sa.Boolean), sa.column("auth_provider", sa.String),
|
||||||
sa.column("password_hash", sa.String), sa.column("password_reset_required", sa.Boolean),
|
sa.column("password_hash", sa.String), sa.column("password_reset_required", sa.Boolean),
|
||||||
@@ -166,54 +166,54 @@ def upgrade() -> None:
|
|||||||
bind.execute(accounts_table.insert(), list(account_rows.values()))
|
bind.execute(accounts_table.insert(), list(account_rows.values()))
|
||||||
for row in users:
|
for row in users:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE users SET account_id = :account_id WHERE id = :user_id"),
|
sa.text("UPDATE access_users SET account_id = :account_id WHERE id = :user_id"),
|
||||||
{"account_id": accounts_by_email[_normalize_email(row["email"])], "user_id": row["id"]},
|
{"account_id": accounts_by_email[_normalize_email(row["email"])], "user_id": row["id"]},
|
||||||
)
|
)
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"UPDATE auth_sessions SET account_id = (SELECT users.account_id FROM users WHERE users.id = auth_sessions.user_id)"
|
"UPDATE access_auth_sessions SET account_id = (SELECT access_users.account_id FROM access_users WHERE access_users.id = access_auth_sessions.user_id)"
|
||||||
))
|
))
|
||||||
|
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
batch_op.create_foreign_key("fk_users_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
batch_op.create_foreign_key("fk_users_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
||||||
batch_op.create_unique_constraint("uq_users_tenant_account", ["tenant_id", "account_id"])
|
batch_op.create_unique_constraint("uq_users_tenant_account", ["tenant_id", "account_id"])
|
||||||
op.create_index(op.f("ix_users_account_id"), "users", ["account_id"])
|
op.create_index(op.f("ix_access_users_account_id"), "access_users", ["account_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE")
|
||||||
op.create_index(op.f("ix_auth_sessions_account_id"), "auth_sessions", ["account_id"])
|
op.create_index(op.f("ix_access_auth_sessions_account_id"), "access_auth_sessions", ["account_id"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"system_role_assignments",
|
"access_system_role_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("account_id", sa.String(length=36), nullable=False),
|
sa.Column("account_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("role_id", sa.String(length=36), nullable=False),
|
sa.Column("role_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["account_id"], ["accounts.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["account_id"], ["access_accounts.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"),
|
sa.UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_system_role_assignments_account_id"), "system_role_assignments", ["account_id"])
|
op.create_index(op.f("ix_access_system_role_assignments_account_id"), "access_system_role_assignments", ["account_id"])
|
||||||
op.create_index(op.f("ix_system_role_assignments_role_id"), "system_role_assignments", ["role_id"])
|
op.create_index(op.f("ix_access_system_role_assignments_role_id"), "access_system_role_assignments", ["role_id"])
|
||||||
|
|
||||||
roles_table = sa.table(
|
roles_table = sa.table(
|
||||||
"roles",
|
"access_roles",
|
||||||
sa.column("id", sa.String), sa.column("tenant_id", sa.String), sa.column("slug", sa.String),
|
sa.column("id", sa.String), sa.column("tenant_id", sa.String), sa.column("slug", sa.String),
|
||||||
sa.column("name", sa.String), sa.column("description", sa.Text), sa.column("permissions", sa.JSON),
|
sa.column("name", sa.String), sa.column("description", sa.Text), sa.column("permissions", sa.JSON),
|
||||||
sa.column("is_builtin", sa.Boolean), sa.column("is_assignable", sa.Boolean),
|
sa.column("is_builtin", sa.Boolean), sa.column("is_assignable", sa.Boolean),
|
||||||
sa.column("created_at", sa.DateTime(timezone=True)), sa.column("updated_at", sa.DateTime(timezone=True)),
|
sa.column("created_at", sa.DateTime(timezone=True)), sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
)
|
)
|
||||||
now = _now()
|
now = _now()
|
||||||
tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenants")).all()]
|
tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenancy_tenants")).all()]
|
||||||
definitions = _role_definitions()
|
definitions = _role_definitions()
|
||||||
tenant_role_ids: dict[tuple[str, str], str] = {}
|
tenant_role_ids: dict[tuple[str, str], str] = {}
|
||||||
for tenant_id in tenant_ids:
|
for tenant_id in tenant_ids:
|
||||||
for slug, definition in definitions.items():
|
for slug, definition in definitions.items():
|
||||||
existing = bind.execute(
|
existing = bind.execute(
|
||||||
sa.text("SELECT id FROM roles WHERE tenant_id = :tenant_id AND slug = :slug"),
|
sa.text("SELECT id FROM access_roles WHERE tenant_id = :tenant_id AND slug = :slug"),
|
||||||
{"tenant_id": tenant_id, "slug": slug},
|
{"tenant_id": tenant_id, "slug": slug},
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing:
|
if existing:
|
||||||
@@ -253,7 +253,7 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
op.create_index(
|
op.create_index(
|
||||||
"uq_roles_system_slug",
|
"uq_roles_system_slug",
|
||||||
"roles",
|
"access_roles",
|
||||||
["slug"],
|
["slug"],
|
||||||
unique=True,
|
unique=True,
|
||||||
sqlite_where=sa.text("tenant_id IS NULL"),
|
sqlite_where=sa.text("tenant_id IS NULL"),
|
||||||
@@ -267,11 +267,11 @@ def upgrade() -> None:
|
|||||||
continue
|
continue
|
||||||
owner_role_id = tenant_role_ids[(row["tenant_id"], "owner")]
|
owner_role_id = tenant_role_ids[(row["tenant_id"], "owner")]
|
||||||
exists = bind.execute(sa.text(
|
exists = bind.execute(sa.text(
|
||||||
"SELECT 1 FROM user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id"
|
"SELECT 1 FROM access_user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id"
|
||||||
), {"tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id}).first()
|
), {"tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id}).first()
|
||||||
if not exists:
|
if not exists:
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"INSERT INTO user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)"
|
"INSERT INTO access_user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)"
|
||||||
), {"id": str(uuid.uuid4()), "tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id, "created_at": now, "updated_at": now})
|
), {"id": str(uuid.uuid4()), "tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id, "created_at": now, "updated_at": now})
|
||||||
|
|
||||||
# Bootstrap rule for existing installations: the earliest active legacy
|
# Bootstrap rule for existing installations: the earliest active legacy
|
||||||
@@ -284,40 +284,40 @@ def upgrade() -> None:
|
|||||||
), None)
|
), None)
|
||||||
if first_system_owner_account_id:
|
if first_system_owner_account_id:
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"INSERT INTO system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)"
|
"INSERT INTO access_system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)"
|
||||||
), {"id": str(uuid.uuid4()), "account_id": first_system_owner_account_id, "role_id": system_owner_role_id, "created_at": now, "updated_at": now})
|
), {"id": str(uuid.uuid4()), "account_id": first_system_owner_account_id, "role_id": system_owner_role_id, "created_at": now, "updated_at": now})
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index("uq_roles_system_slug", table_name="roles")
|
op.drop_index("uq_roles_system_slug", table_name="access_roles")
|
||||||
op.drop_index(op.f("ix_system_role_assignments_role_id"), table_name="system_role_assignments")
|
op.drop_index(op.f("ix_access_system_role_assignments_role_id"), table_name="access_system_role_assignments")
|
||||||
op.drop_index(op.f("ix_system_role_assignments_account_id"), table_name="system_role_assignments")
|
op.drop_index(op.f("ix_access_system_role_assignments_account_id"), table_name="access_system_role_assignments")
|
||||||
op.drop_table("system_role_assignments")
|
op.drop_table("access_system_role_assignments")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_auth_sessions_account_id"), table_name="auth_sessions")
|
op.drop_index(op.f("ix_access_auth_sessions_account_id"), table_name="access_auth_sessions")
|
||||||
with op.batch_alter_table("auth_sessions") as batch_op:
|
with op.batch_alter_table("access_auth_sessions") as batch_op:
|
||||||
batch_op.drop_constraint("fk_auth_sessions_account_id_accounts", type_="foreignkey")
|
batch_op.drop_constraint("fk_auth_sessions_account_id_accounts", type_="foreignkey")
|
||||||
batch_op.drop_column("account_id")
|
batch_op.drop_column("account_id")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_users_account_id"), table_name="users")
|
op.drop_index(op.f("ix_access_users_account_id"), table_name="access_users")
|
||||||
with op.batch_alter_table("users") as batch_op:
|
with op.batch_alter_table("access_users") as batch_op:
|
||||||
batch_op.drop_constraint("uq_users_tenant_account", type_="unique")
|
batch_op.drop_constraint("uq_users_tenant_account", type_="unique")
|
||||||
batch_op.drop_constraint("fk_users_account_id_accounts", type_="foreignkey")
|
batch_op.drop_constraint("fk_users_account_id_accounts", type_="foreignkey")
|
||||||
batch_op.drop_column("account_id")
|
batch_op.drop_column("account_id")
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.drop_column("is_assignable")
|
batch_op.drop_column("is_assignable")
|
||||||
batch_op.drop_column("is_builtin")
|
batch_op.drop_column("is_builtin")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.drop_column("is_active")
|
batch_op.drop_column("is_active")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.drop_column("settings")
|
batch_op.drop_column("settings")
|
||||||
batch_op.drop_column("default_locale")
|
batch_op.drop_column("default_locale")
|
||||||
batch_op.drop_column("description")
|
batch_op.drop_column("description")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_accounts_normalized_email"), table_name="accounts")
|
op.drop_index(op.f("ix_access_accounts_normalized_email"), table_name="access_accounts")
|
||||||
op.drop_table("accounts")
|
op.drop_table("access_accounts")
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ def upgrade() -> None:
|
|||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
# Reconcile only the empty create_all shape for the newly introduced tables.
|
# Reconcile only the empty create_all shape for the newly introduced tables.
|
||||||
for table_name in ("governance_template_assignments", "governance_templates", "system_settings"):
|
for table_name in ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings"):
|
||||||
if table_name in tables:
|
if table_name in tables:
|
||||||
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()
|
||||||
if count:
|
if count:
|
||||||
@@ -36,7 +36,7 @@ def upgrade() -> None:
|
|||||||
op.drop_table(table_name)
|
op.drop_table(table_name)
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"system_settings",
|
"core_system_settings",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"),
|
sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"),
|
||||||
sa.Column("allow_tenant_custom_groups", sa.Boolean(), nullable=False, server_default=sa.true()),
|
sa.Column("allow_tenant_custom_groups", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||||
@@ -50,16 +50,23 @@ def upgrade() -> None:
|
|||||||
now = _now()
|
now = _now()
|
||||||
bind.execute(sa.text(
|
bind.execute(sa.text(
|
||||||
"""
|
"""
|
||||||
INSERT INTO system_settings
|
INSERT INTO core_system_settings
|
||||||
(id, default_locale, allow_tenant_custom_groups, allow_tenant_custom_roles,
|
(id, default_locale, allow_tenant_custom_groups, allow_tenant_custom_roles,
|
||||||
allow_tenant_api_keys, settings, created_at, updated_at)
|
allow_tenant_api_keys, settings, created_at, updated_at)
|
||||||
VALUES
|
VALUES
|
||||||
('global', 'en', 1, 1, 1, '{}', :created_at, :updated_at)
|
('global', 'en', :allow_tenant_custom_groups, :allow_tenant_custom_roles,
|
||||||
|
:allow_tenant_api_keys, '{}', :created_at, :updated_at)
|
||||||
"""
|
"""
|
||||||
), {"created_at": now, "updated_at": now})
|
), {
|
||||||
|
"allow_tenant_custom_groups": True,
|
||||||
|
"allow_tenant_custom_roles": True,
|
||||||
|
"allow_tenant_api_keys": True,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
})
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"governance_templates",
|
"admin_governance_templates",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("kind", sa.String(length=20), nullable=False),
|
sa.Column("kind", sa.String(length=20), nullable=False),
|
||||||
sa.Column("slug", sa.String(length=100), nullable=False),
|
sa.Column("slug", sa.String(length=100), nullable=False),
|
||||||
@@ -72,51 +79,51 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"),
|
sa.UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_governance_templates_kind"), "governance_templates", ["kind"])
|
op.create_index(op.f("ix_admin_governance_templates_kind"), "admin_governance_templates", ["kind"])
|
||||||
|
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"governance_template_assignments",
|
"admin_governance_template_assignments",
|
||||||
sa.Column("id", sa.String(length=36), nullable=False),
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("template_id", sa.String(length=36), nullable=False),
|
sa.Column("template_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
sa.Column("mode", sa.String(length=20), nullable=False, server_default="available"),
|
sa.Column("mode", sa.String(length=20), nullable=False, server_default="available"),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["template_id"], ["governance_templates.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["template_id"], ["admin_governance_templates.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"),
|
sa.UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"),
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_governance_template_assignments_template_id"), "governance_template_assignments", ["template_id"])
|
op.create_index(op.f("ix_admin_governance_template_assignments_template_id"), "admin_governance_template_assignments", ["template_id"])
|
||||||
op.create_index(op.f("ix_governance_template_assignments_tenant_id"), "governance_template_assignments", ["tenant_id"])
|
op.create_index(op.f("ix_admin_governance_template_assignments_tenant_id"), "admin_governance_template_assignments", ["tenant_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.add_column(sa.Column("allow_custom_groups", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_custom_groups", sa.Boolean(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("allow_custom_roles", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_custom_roles", sa.Boolean(), nullable=True))
|
||||||
batch_op.add_column(sa.Column("allow_api_keys", sa.Boolean(), nullable=True))
|
batch_op.add_column(sa.Column("allow_api_keys", sa.Boolean(), nullable=True))
|
||||||
|
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_groups_system_template_id_governance_templates",
|
"fk_groups_system_template_id_governance_templates",
|
||||||
"governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
"admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_groups_system_template_id"), "groups", ["system_template_id"])
|
op.create_index(op.f("ix_access_groups_system_template_id"), "access_groups", ["system_template_id"])
|
||||||
|
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True))
|
||||||
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||||
batch_op.create_foreign_key(
|
batch_op.create_foreign_key(
|
||||||
"fk_roles_system_template_id_governance_templates",
|
"fk_roles_system_template_id_governance_templates",
|
||||||
"governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
"admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL",
|
||||||
)
|
)
|
||||||
op.create_index(op.f("ix_roles_system_template_id"), "roles", ["system_template_id"])
|
op.create_index(op.f("ix_access_roles_system_template_id"), "access_roles", ["system_template_id"])
|
||||||
|
|
||||||
# Existing system owners use system:* and need no data change. Extend the
|
# Existing system owners use system:* and need no data change. Extend the
|
||||||
# read-only built-in auditor role to the newly introduced read scopes.
|
# read-only built-in auditor role to the newly introduced read scopes.
|
||||||
auditor = bind.execute(sa.text(
|
auditor = bind.execute(sa.text(
|
||||||
"SELECT id, permissions FROM roles WHERE tenant_id IS NULL AND slug = 'system_auditor'"
|
"SELECT id, permissions FROM access_roles WHERE tenant_id IS NULL AND slug = 'system_auditor'"
|
||||||
)).mappings().first()
|
)).mappings().first()
|
||||||
if auditor:
|
if auditor:
|
||||||
raw_permissions = auditor["permissions"] or []
|
raw_permissions = auditor["permissions"] or []
|
||||||
@@ -125,7 +132,7 @@ def upgrade() -> None:
|
|||||||
if scope not in permissions:
|
if scope not in permissions:
|
||||||
permissions.append(scope)
|
permissions.append(scope)
|
||||||
roles_table = sa.table(
|
roles_table = sa.table(
|
||||||
"roles",
|
"access_roles",
|
||||||
sa.column("id", sa.String),
|
sa.column("id", sa.String),
|
||||||
sa.column("permissions", sa.JSON),
|
sa.column("permissions", sa.JSON),
|
||||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||||
@@ -138,26 +145,26 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_index(op.f("ix_roles_system_template_id"), table_name="roles")
|
op.drop_index(op.f("ix_access_roles_system_template_id"), table_name="access_roles")
|
||||||
with op.batch_alter_table("roles") as batch_op:
|
with op.batch_alter_table("access_roles") as batch_op:
|
||||||
batch_op.drop_constraint("fk_roles_system_template_id_governance_templates", type_="foreignkey")
|
batch_op.drop_constraint("fk_roles_system_template_id_governance_templates", type_="foreignkey")
|
||||||
batch_op.drop_column("system_required")
|
batch_op.drop_column("system_required")
|
||||||
batch_op.drop_column("system_template_id")
|
batch_op.drop_column("system_template_id")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_groups_system_template_id"), table_name="groups")
|
op.drop_index(op.f("ix_access_groups_system_template_id"), table_name="access_groups")
|
||||||
with op.batch_alter_table("groups") as batch_op:
|
with op.batch_alter_table("access_groups") as batch_op:
|
||||||
batch_op.drop_constraint("fk_groups_system_template_id_governance_templates", type_="foreignkey")
|
batch_op.drop_constraint("fk_groups_system_template_id_governance_templates", type_="foreignkey")
|
||||||
batch_op.drop_column("system_required")
|
batch_op.drop_column("system_required")
|
||||||
batch_op.drop_column("system_template_id")
|
batch_op.drop_column("system_template_id")
|
||||||
|
|
||||||
with op.batch_alter_table("tenants") as batch_op:
|
with op.batch_alter_table("tenancy_tenants") as batch_op:
|
||||||
batch_op.drop_column("allow_api_keys")
|
batch_op.drop_column("allow_api_keys")
|
||||||
batch_op.drop_column("allow_custom_roles")
|
batch_op.drop_column("allow_custom_roles")
|
||||||
batch_op.drop_column("allow_custom_groups")
|
batch_op.drop_column("allow_custom_groups")
|
||||||
|
|
||||||
op.drop_index(op.f("ix_governance_template_assignments_tenant_id"), table_name="governance_template_assignments")
|
op.drop_index(op.f("ix_admin_governance_template_assignments_tenant_id"), table_name="admin_governance_template_assignments")
|
||||||
op.drop_index(op.f("ix_governance_template_assignments_template_id"), table_name="governance_template_assignments")
|
op.drop_index(op.f("ix_admin_governance_template_assignments_template_id"), table_name="admin_governance_template_assignments")
|
||||||
op.drop_table("governance_template_assignments")
|
op.drop_table("admin_governance_template_assignments")
|
||||||
op.drop_index(op.f("ix_governance_templates_kind"), table_name="governance_templates")
|
op.drop_index(op.f("ix_admin_governance_templates_kind"), table_name="admin_governance_templates")
|
||||||
op.drop_table("governance_templates")
|
op.drop_table("admin_governance_templates")
|
||||||
op.drop_table("system_settings")
|
op.drop_table("core_system_settings")
|
||||||
|
|||||||
@@ -148,9 +148,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["campaign_id"], ["campaigns.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["campaign_id"], ["campaigns.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("campaign_id", "target_type", "target_id", name="uq_campaign_share_target"),
|
sa.UniqueConstraint("campaign_id", "target_type", "target_id", name="uq_campaign_share_target"),
|
||||||
)
|
)
|
||||||
@@ -160,32 +160,32 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_campaign_shares_target_id", "campaign_shares", ["target_id"])
|
op.create_index("ix_campaign_shares_target_id", "campaign_shares", ["target_id"])
|
||||||
op.create_index("ix_campaign_shares_created_by_user_id", "campaign_shares", ["created_by_user_id"])
|
op.create_index("ix_campaign_shares_created_by_user_id", "campaign_shares", ["created_by_user_id"])
|
||||||
op.create_index("ix_campaign_shares_revoked_at", "campaign_shares", ["revoked_at"])
|
op.create_index("ix_campaign_shares_revoked_at", "campaign_shares", ["revoked_at"])
|
||||||
if "roles" not in tables:
|
if "access_roles" not in tables:
|
||||||
return
|
return
|
||||||
|
|
||||||
rows = bind.execute(sa.text("SELECT id, permissions FROM roles")).mappings().all()
|
rows = bind.execute(sa.text("SELECT id, permissions FROM access_roles")).mappings().all()
|
||||||
for row in rows:
|
for row in rows:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE roles SET permissions = :permissions WHERE id = :id"),
|
sa.text("UPDATE access_roles SET permissions = :permissions WHERE id = :id"),
|
||||||
{"id": row["id"], "permissions": _json(_expand_legacy(_decode(row["permissions"])))},
|
{"id": row["id"], "permissions": _json(_expand_legacy(_decode(row["permissions"])))},
|
||||||
)
|
)
|
||||||
|
|
||||||
for slug, permissions in TENANT_ROLE_PERMISSIONS.items():
|
for slug, permissions in TENANT_ROLE_PERMISSIONS.items():
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text("UPDATE roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"),
|
sa.text("UPDATE access_roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"),
|
||||||
{"slug": slug, "permissions": _json(permissions)},
|
{"slug": slug, "permissions": _json(permissions)},
|
||||||
)
|
)
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
for slug, permissions in SYSTEM_ROLE_PERMISSIONS.items():
|
for slug, permissions in SYSTEM_ROLE_PERMISSIONS.items():
|
||||||
existing = bind.execute(
|
existing = bind.execute(
|
||||||
sa.text("SELECT id FROM roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug}
|
sa.text("SELECT id FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug}
|
||||||
).first()
|
).first()
|
||||||
is_protected = slug == "system_owner"
|
is_protected = slug == "system_owner"
|
||||||
if existing:
|
if existing:
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"UPDATE roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE "
|
"UPDATE access_roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE "
|
||||||
"WHERE tenant_id IS NULL AND slug = :slug"
|
"WHERE tenant_id IS NULL AND slug = :slug"
|
||||||
),
|
),
|
||||||
{"slug": slug, "permissions": _json(permissions), "is_builtin": is_protected},
|
{"slug": slug, "permissions": _json(permissions), "is_builtin": is_protected},
|
||||||
@@ -199,7 +199,7 @@ def upgrade() -> None:
|
|||||||
name, description = names[slug]
|
name, description = names[slug]
|
||||||
bind.execute(
|
bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"INSERT INTO roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, "
|
"INSERT INTO access_roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, "
|
||||||
"system_template_id, system_required, created_at, updated_at) "
|
"system_template_id, system_required, created_at, updated_at) "
|
||||||
"VALUES (:id, NULL, :slug, :name, :description, :permissions, :is_builtin, TRUE, NULL, FALSE, :created_at, :updated_at)"
|
"VALUES (:id, NULL, :slug, :name, :description, :permissions, :is_builtin, TRUE, NULL, FALSE, :created_at, :updated_at)"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ depends_on = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# ### commands auto generated by Alembic - please adjust! ###
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
op.create_table('tenants',
|
op.create_table('tenancy_tenants',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
@@ -27,7 +27,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tenants'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_tenants'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_tenants_slug'), 'tenants', ['slug'], unique=True)
|
op.create_index(op.f('ix_tenancy_tenants_slug'), 'tenancy_tenants', ['slug'], unique=True)
|
||||||
op.create_table('attachment_blobs',
|
op.create_table('attachment_blobs',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
@@ -38,25 +38,25 @@ def upgrade() -> None:
|
|||||||
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
sa.Column('storage_key', sa.String(length=1000), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')),
|
||||||
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False)
|
||||||
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False)
|
||||||
op.create_table('groups',
|
op.create_table('access_groups',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_groups')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_groups')),
|
||||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_groups_tenant_id'), 'groups', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False)
|
||||||
op.create_table('roles',
|
op.create_table('access_roles',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
sa.Column('tenant_id', sa.String(length=36), nullable=True),
|
||||||
sa.Column('slug', sa.String(length=100), nullable=False),
|
sa.Column('slug', sa.String(length=100), nullable=False),
|
||||||
@@ -64,12 +64,12 @@ def upgrade() -> None:
|
|||||||
sa.Column('permissions', sa.JSON(), nullable=False),
|
sa.Column('permissions', sa.JSON(), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_roles')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_roles')),
|
||||||
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_roles_tenant_id'), 'roles', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False)
|
||||||
op.create_table('users',
|
op.create_table('access_users',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('email', sa.String(length=320), nullable=False),
|
sa.Column('email', sa.String(length=320), nullable=False),
|
||||||
@@ -78,13 +78,13 @@ def upgrade() -> None:
|
|||||||
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
sa.Column('is_tenant_admin', sa.Boolean(), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_users')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_users')),
|
||||||
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=False)
|
op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False)
|
||||||
op.create_index(op.f('ix_users_tenant_id'), 'users', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False)
|
||||||
op.create_table('api_keys',
|
op.create_table('access_api_keys',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('user_id', sa.String(length=36), nullable=False),
|
sa.Column('user_id', sa.String(length=36), nullable=False),
|
||||||
@@ -97,13 +97,13 @@ def upgrade() -> None:
|
|||||||
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_api_keys'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_api_keys'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_api_keys_prefix'), 'api_keys', ['prefix'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False)
|
||||||
op.create_index(op.f('ix_api_keys_tenant_id'), 'api_keys', ['tenant_id'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False)
|
||||||
op.create_index(op.f('ix_api_keys_user_id'), 'api_keys', ['user_id'], unique=False)
|
op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False)
|
||||||
op.create_table('campaigns',
|
op.create_table('campaigns',
|
||||||
sa.Column('id', sa.String(length=36), nullable=False),
|
sa.Column('id', sa.String(length=36), nullable=False),
|
||||||
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
sa.Column('tenant_id', sa.String(length=36), nullable=False),
|
||||||
@@ -115,8 +115,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
sa.Column('current_version_id', sa.String(length=36), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['created_by_user_id'], ['users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')),
|
||||||
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id')
|
||||||
)
|
)
|
||||||
@@ -138,8 +138,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['owner_user_id'], ['users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False)
|
||||||
@@ -157,9 +157,9 @@ def upgrade() -> None:
|
|||||||
sa.Column('details', sa.JSON(), nullable=True),
|
sa.Column('details', sa.JSON(), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['api_key_id'], ['api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False)
|
||||||
@@ -213,7 +213,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')),
|
||||||
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry')
|
||||||
)
|
)
|
||||||
@@ -243,7 +243,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False)
|
||||||
@@ -327,20 +327,20 @@ def downgrade() -> None:
|
|||||||
op.drop_index(op.f('ix_campaigns_external_id'), table_name='campaigns')
|
op.drop_index(op.f('ix_campaigns_external_id'), table_name='campaigns')
|
||||||
op.drop_index(op.f('ix_campaigns_created_by_user_id'), table_name='campaigns')
|
op.drop_index(op.f('ix_campaigns_created_by_user_id'), table_name='campaigns')
|
||||||
op.drop_table('campaigns')
|
op.drop_table('campaigns')
|
||||||
op.drop_index(op.f('ix_api_keys_user_id'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_user_id'), table_name='access_api_keys')
|
||||||
op.drop_index(op.f('ix_api_keys_tenant_id'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_tenant_id'), table_name='access_api_keys')
|
||||||
op.drop_index(op.f('ix_api_keys_prefix'), table_name='api_keys')
|
op.drop_index(op.f('ix_access_api_keys_prefix'), table_name='access_api_keys')
|
||||||
op.drop_table('api_keys')
|
op.drop_table('access_api_keys')
|
||||||
op.drop_index(op.f('ix_users_tenant_id'), table_name='users')
|
op.drop_index(op.f('ix_access_users_tenant_id'), table_name='access_users')
|
||||||
op.drop_index(op.f('ix_users_email'), table_name='users')
|
op.drop_index(op.f('ix_access_users_email'), table_name='access_users')
|
||||||
op.drop_table('users')
|
op.drop_table('access_users')
|
||||||
op.drop_index(op.f('ix_roles_tenant_id'), table_name='roles')
|
op.drop_index(op.f('ix_access_roles_tenant_id'), table_name='access_roles')
|
||||||
op.drop_table('roles')
|
op.drop_table('access_roles')
|
||||||
op.drop_index(op.f('ix_groups_tenant_id'), table_name='groups')
|
op.drop_index(op.f('ix_access_groups_tenant_id'), table_name='access_groups')
|
||||||
op.drop_table('groups')
|
op.drop_table('access_groups')
|
||||||
op.drop_index(op.f('ix_attachment_blobs_tenant_id'), table_name='attachment_blobs')
|
op.drop_index(op.f('ix_attachment_blobs_tenant_id'), table_name='attachment_blobs')
|
||||||
op.drop_index(op.f('ix_attachment_blobs_sha256'), table_name='attachment_blobs')
|
op.drop_index(op.f('ix_attachment_blobs_sha256'), table_name='attachment_blobs')
|
||||||
op.drop_table('attachment_blobs')
|
op.drop_table('attachment_blobs')
|
||||||
op.drop_index(op.f('ix_tenants_slug'), table_name='tenants')
|
op.drop_index(op.f('ix_tenancy_tenants_slug'), table_name='tenancy_tenants')
|
||||||
op.drop_table('tenants')
|
op.drop_table('tenancy_tenants')
|
||||||
# ### end Alembic commands ###
|
# ### end Alembic commands ###
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
if "auth_sessions" in tables:
|
if "access_auth_sessions" in tables:
|
||||||
columns = {column["name"] for column in inspector.get_columns("auth_sessions")}
|
columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")}
|
||||||
if "csrf_token_hash" not in columns:
|
if "csrf_token_hash" not in columns:
|
||||||
op.add_column("auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True))
|
op.add_column("access_auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True))
|
||||||
if "mail_server_profiles" not in tables:
|
if "mail_server_profiles" not in tables:
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"mail_server_profiles",
|
"mail_server_profiles",
|
||||||
@@ -40,9 +40,9 @@ def upgrade() -> None:
|
|||||||
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
|
sa.Column("updated_by_user_id", sa.String(length=36), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["updated_by_user_id"], ["users.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], ondelete="SET NULL"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("tenant_id", "slug", name="uq_mail_server_profiles_tenant_slug"),
|
sa.UniqueConstraint("tenant_id", "slug", name="uq_mail_server_profiles_tenant_slug"),
|
||||||
)
|
)
|
||||||
@@ -67,7 +67,7 @@ def downgrade() -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
op.drop_table("mail_server_profiles")
|
op.drop_table("mail_server_profiles")
|
||||||
if "auth_sessions" in inspector.get_table_names():
|
if "access_auth_sessions" in inspector.get_table_names():
|
||||||
columns = {column["name"] for column in inspector.get_columns("auth_sessions")}
|
columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")}
|
||||||
if "csrf_token_hash" in columns:
|
if "csrf_token_hash" in columns:
|
||||||
op.drop_column("auth_sessions", "csrf_token_hash")
|
op.drop_column("access_auth_sessions", "csrf_token_hash")
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ def upgrade() -> None:
|
|||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
|
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name not in tables:
|
if table_name not in tables:
|
||||||
continue
|
continue
|
||||||
columns = _columns(inspector, table_name)
|
columns = _columns(inspector, table_name)
|
||||||
@@ -83,6 +83,6 @@ def downgrade() -> None:
|
|||||||
batch.drop_column("scope_type")
|
batch.drop_column("scope_type")
|
||||||
batch.alter_column("tenant_id", existing_type=sa.String(length=36), nullable=False)
|
batch.alter_column("tenant_id", existing_type=sa.String(length=36), nullable=False)
|
||||||
|
|
||||||
for table_name in ("campaigns", "groups", "users"):
|
for table_name in ("campaigns", "access_groups", "access_users"):
|
||||||
if table_name in tables and "mail_profile_policy" in _columns(inspector, table_name):
|
if table_name in tables and "mail_profile_policy" in _columns(inspector, table_name):
|
||||||
op.drop_column(table_name, "mail_profile_policy")
|
op.drop_column(table_name, "mail_profile_policy")
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def upgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("users", "groups", "campaigns"):
|
for table_name in ("access_users", "access_groups", "campaigns"):
|
||||||
if table_name not in tables:
|
if table_name not in tables:
|
||||||
continue
|
continue
|
||||||
if "settings" not in _columns(inspector, table_name):
|
if "settings" not in _columns(inspector, table_name):
|
||||||
@@ -36,6 +36,6 @@ def downgrade() -> None:
|
|||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
inspector = sa.inspect(bind)
|
inspector = sa.inspect(bind)
|
||||||
tables = set(inspector.get_table_names())
|
tables = set(inspector.get_table_names())
|
||||||
for table_name in ("campaigns", "groups", "users"):
|
for table_name in ("campaigns", "access_groups", "access_users"):
|
||||||
if table_name in tables and "settings" in _columns(inspector, table_name):
|
if table_name in tables and "settings" in _columns(inspector, table_name):
|
||||||
op.drop_column(table_name, "settings")
|
op.drop_column(table_name, "settings")
|
||||||
|
|||||||
5
dev/postgres/.env.example
Normal file
5
dev/postgres/.env.example
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
GOVOPLAN_POSTGRES_DB=govoplan
|
||||||
|
GOVOPLAN_POSTGRES_USER=govoplan
|
||||||
|
GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev
|
||||||
|
GOVOPLAN_POSTGRES_PORT=55432
|
||||||
|
GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan
|
||||||
126
dev/postgres/README.md
Normal file
126
dev/postgres/README.md
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
# PostgreSQL Development Profile
|
||||||
|
|
||||||
|
GovOPlaN development now defaults to PostgreSQL:
|
||||||
|
|
||||||
|
```text
|
||||||
|
postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||||
|
```
|
||||||
|
|
||||||
|
The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Store the password in `~/.pgpass` instead of committing it to a `.env` file.
|
||||||
|
The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if
|
||||||
|
you need a different database.
|
||||||
|
|
||||||
|
## Host PostgreSQL Setup
|
||||||
|
|
||||||
|
Create the default local role and database from a host shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export GOVOPLAN_PG_DB=govoplan_dev
|
||||||
|
export GOVOPLAN_PG_USER=govoplan_dev
|
||||||
|
read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD
|
||||||
|
echo
|
||||||
|
|
||||||
|
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then
|
||||||
|
sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}"
|
||||||
|
else
|
||||||
|
sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then
|
||||||
|
sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};"
|
||||||
|
|
||||||
|
touch ~/.pgpass
|
||||||
|
chmod 600 ~/.pgpass
|
||||||
|
grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true
|
||||||
|
printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp
|
||||||
|
mv ~/.pgpass.tmp ~/.pgpass
|
||||||
|
chmod 600 ~/.pgpass
|
||||||
|
```
|
||||||
|
|
||||||
|
Check access:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \
|
||||||
|
-c 'select current_user, current_database();'
|
||||||
|
```
|
||||||
|
|
||||||
|
When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are
|
||||||
|
available through:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
flatpak-spawn --host /usr/bin/psql --version
|
||||||
|
flatpak-spawn --host /usr/bin/pg_dump --version
|
||||||
|
flatpak-spawn --host /usr/bin/pg_restore --version
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run GovOPlaN Against Host PostgreSQL
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||||
|
--with-dev-data
|
||||||
|
|
||||||
|
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
For the full backend and WebUI launcher:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
To force the old SQLite fallback for a disposable local run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Disposable Docker Testbed
|
||||||
|
|
||||||
|
This testbed is for migration and module-permutation checks. It is not a
|
||||||
|
production deployment profile.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the integration check from the core checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
`--reset-schema` drops and recreates the `public` schema before every module
|
||||||
|
set. Use it only against this disposable database.
|
||||||
|
|
||||||
|
Stop the testbed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
docker compose --env-file .env down
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove all test data:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env down -v
|
||||||
|
```
|
||||||
22
dev/postgres/docker-compose.yml
Normal file
22
dev/postgres/docker-compose.yml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: govoplan-core-postgres-dev
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan}
|
||||||
|
POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan}
|
||||||
|
POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev}
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD-SHELL
|
||||||
|
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres-data:
|
||||||
9
dev/production-like/.env.example
Normal file
9
dev/production-like/.env.example
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379
|
||||||
|
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0
|
||||||
49
dev/production-like/README.md
Normal file
49
dev/production-like/README.md
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
# Production-Like Development Profile
|
||||||
|
|
||||||
|
This profile runs the shared services that production depends on while keeping
|
||||||
|
API, worker, and WebUI code in the editable local repositories.
|
||||||
|
|
||||||
|
It provides:
|
||||||
|
|
||||||
|
- PostgreSQL with a persistent Docker volume
|
||||||
|
- Redis with append-only persistence
|
||||||
|
- explicit `ENABLED_MODULES`
|
||||||
|
- local durable file storage under `runtime/production-like/files`
|
||||||
|
- a Celery worker process using the same queues as the API
|
||||||
|
|
||||||
|
Start it from the core repository:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The launcher uses `dev/production-like/.env` when present, otherwise
|
||||||
|
`dev/production-like/.env.example`. Copy the example when you want local port or
|
||||||
|
password changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp dev/production-like/.env.example dev/production-like/.env
|
||||||
|
```
|
||||||
|
|
||||||
|
The API and worker use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan
|
||||||
|
REDIS_URL=redis://127.0.0.1:56379/0
|
||||||
|
CELERY_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis
|
||||||
|
containers keep running by default so the next launch is fast. To stop them too:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
To remove all profile data:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/production-like
|
||||||
|
docker compose --env-file .env.example down -v
|
||||||
|
```
|
||||||
37
dev/production-like/docker-compose.yml
Normal file
37
dev/production-like/docker-compose.yml
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: govoplan-production-like-postgres
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan}
|
||||||
|
POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan}
|
||||||
|
POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev}
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD-SHELL
|
||||||
|
- pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
container_name: govoplan-production-like-redis
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
volumes:
|
||||||
|
- redis-data:/data
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres-data:
|
||||||
|
redis-data:
|
||||||
@@ -1,456 +0,0 @@
|
|||||||
# GovOPlaN Access Extraction Plan
|
|
||||||
|
|
||||||
> Backlog state migrated to Gitea issues on 2026-07-06. Keep this document as
|
|
||||||
> durable extraction context and architecture notes. Track active tasks,
|
|
||||||
> decisions, blockers, and implementation state in `add-ideas/govoplan-core`
|
|
||||||
> issues with `module/access` and `source/backlog-import`.
|
|
||||||
|
|
||||||
This plan describes how to extract access, authentication, RBAC, and related
|
|
||||||
administration behavior from the current compatibility core into a dedicated
|
|
||||||
`govoplan-access` platform module.
|
|
||||||
|
|
||||||
The goal is not to make access optional in every real deployment. The goal is to
|
|
||||||
make ownership explicit: the kernel composes modules and exposes contracts;
|
|
||||||
`govoplan-access` owns identity, sessions, API keys, roles, groups, and access
|
|
||||||
administration.
|
|
||||||
|
|
||||||
## Current Inventory
|
|
||||||
|
|
||||||
### Existing Access Module Seed
|
|
||||||
|
|
||||||
`govoplan-access` contains the extracted module-shaped seed package under
|
|
||||||
`src/govoplan_access/backend`. `govoplan-core` keeps compatibility import shims
|
|
||||||
under `src/govoplan_core/access`:
|
|
||||||
|
|
||||||
- `manifest.py` declares `ModuleManifest(id="access")`.
|
|
||||||
- `db/models.py` defines access-owned model candidates.
|
|
||||||
- `auth/principals.py`, `auth/roles.py`, and `auth/tokens.py` contain auth
|
|
||||||
helper concepts.
|
|
||||||
- `permissions/definitions.py`, `permissions/evaluator.py`, and
|
|
||||||
`permissions/registry.py` contain permission catalogue/evaluation concepts.
|
|
||||||
- `tenancy/datastore.py` contains early tenancy access helpers.
|
|
||||||
|
|
||||||
This seed package is now the starting point for the access module, but it is
|
|
||||||
not yet the full live source of truth for the running product.
|
|
||||||
|
|
||||||
### Live Compatibility Ownership In Core
|
|
||||||
|
|
||||||
The active implementation still uses core-owned models and services:
|
|
||||||
|
|
||||||
- `src/govoplan_core/db/models.py`
|
|
||||||
- `Account`
|
|
||||||
- `Tenant`
|
|
||||||
- `User`
|
|
||||||
- `Group`
|
|
||||||
- `Role`
|
|
||||||
- `SystemSettings`
|
|
||||||
- `GovernanceTemplate`
|
|
||||||
- `GovernanceTemplateAssignment`
|
|
||||||
- `SystemRoleAssignment`
|
|
||||||
- `UserGroupMembership`
|
|
||||||
- `UserRoleAssignment`
|
|
||||||
- `GroupRoleAssignment`
|
|
||||||
- `ApiKey`
|
|
||||||
- `AuthSession`
|
|
||||||
- `AuditLog`
|
|
||||||
- `src/govoplan_core/auth/dependencies.py`
|
|
||||||
- `src/govoplan_core/security/api_keys.py`
|
|
||||||
- `src/govoplan_core/security/permissions.py`
|
|
||||||
- `src/govoplan_core/security/sessions.py`
|
|
||||||
- `src/govoplan_core/security/passwords.py`
|
|
||||||
- `src/govoplan_core/security/secrets.py`
|
|
||||||
- `src/govoplan_core/security/module_permissions.py`
|
|
||||||
- `src/govoplan_core/admin/service.py`
|
|
||||||
- `src/govoplan_core/admin/governance.py`
|
|
||||||
- `src/govoplan_core/api/v1/auth.py`
|
|
||||||
- `src/govoplan_core/api/v1/admin.py`
|
|
||||||
- `src/govoplan_core/api/v1/admin_schemas.py`
|
|
||||||
- `src/govoplan_core/api/v1/audit.py`
|
|
||||||
- `src/govoplan_core/db/bootstrap.py`
|
|
||||||
|
|
||||||
Core still hosts the generic login/settings shell and shared WebUI primitives.
|
|
||||||
The legacy administration page has moved to the `govoplan-access` WebUI package
|
|
||||||
and is contributed as the `/admin` route by the access module. Individual admin
|
|
||||||
panels can now be split further into access, admin, tenancy, policy, and audit
|
|
||||||
WebUI contributions without changing the core shell route wiring again.
|
|
||||||
|
|
||||||
### Current Module Consumers
|
|
||||||
|
|
||||||
Feature modules no longer import core auth dependency wrappers or access-owned
|
|
||||||
ORM models. Backend routers import the access-published FastAPI dependency API
|
|
||||||
from `govoplan_access.backend.auth.dependencies`; runtime cooperation uses
|
|
||||||
kernel capabilities such as `access.directory`, `campaigns.access`,
|
|
||||||
`campaigns.mailPolicyContext`, and `campaigns.deliveryTasks`.
|
|
||||||
|
|
||||||
## Target Ownership
|
|
||||||
|
|
||||||
### Kernel
|
|
||||||
|
|
||||||
The kernel keeps only platform composition and stable contracts:
|
|
||||||
|
|
||||||
- module discovery and registry validation
|
|
||||||
- route aggregation
|
|
||||||
- migration orchestration
|
|
||||||
- database engine/session lifecycle
|
|
||||||
- permission catalogue aggregation
|
|
||||||
- capability registry
|
|
||||||
- event/command envelopes
|
|
||||||
- dependency injection hooks
|
|
||||||
- health and diagnostics
|
|
||||||
- compatibility facades while modules migrate
|
|
||||||
|
|
||||||
The kernel should not own account, tenant, role, group, session, or API-key
|
|
||||||
semantics.
|
|
||||||
|
|
||||||
### `govoplan-access`
|
|
||||||
|
|
||||||
`govoplan-access` owns:
|
|
||||||
|
|
||||||
- accounts
|
|
||||||
- authentication routes
|
|
||||||
- session lifecycle
|
|
||||||
- API keys
|
|
||||||
- users
|
|
||||||
- groups and memberships
|
|
||||||
- roles and assignments
|
|
||||||
- principal resolution
|
|
||||||
- permission evaluation
|
|
||||||
- access administration routes
|
|
||||||
- access administration WebUI contributions
|
|
||||||
- access-related migrations
|
|
||||||
- access permissions and role templates
|
|
||||||
|
|
||||||
### Later Platform Modules
|
|
||||||
|
|
||||||
Some current access-adjacent behavior can remain in access initially, but should
|
|
||||||
have clean seams for later extraction:
|
|
||||||
|
|
||||||
- `govoplan-tenancy`: tenants, tenant lifecycle, tenant switching, tenant
|
|
||||||
metadata, tenant settings boundaries.
|
|
||||||
- `govoplan-policy`: hierarchical policy/effective policy resolution and
|
|
||||||
provenance display contracts.
|
|
||||||
- `govoplan-audit`: audit log storage, audit routes, retention hooks, evidence
|
|
||||||
exports.
|
|
||||||
- `govoplan-admin`: generic administration shell contributions if they are not
|
|
||||||
owned by access/tenancy/policy/audit directly.
|
|
||||||
|
|
||||||
## Kernel Contracts To Stabilize First
|
|
||||||
|
|
||||||
Before moving live code, define contracts that feature modules can depend on
|
|
||||||
without importing access ORM models:
|
|
||||||
|
|
||||||
- `PrincipalResolver`
|
|
||||||
- resolves the current actor from a request/session/API key.
|
|
||||||
- returns a stable DTO, not an ORM model.
|
|
||||||
- `AccessDirectory`
|
|
||||||
- resolves users, groups, memberships, and display labels by stable IDs.
|
|
||||||
- supports batch lookups for grids and policy screens.
|
|
||||||
- `TenantResolver`
|
|
||||||
- resolves current tenant context and tenant metadata by stable ID.
|
|
||||||
- `PermissionEvaluator`
|
|
||||||
- evaluates required scopes for a principal and resource.
|
|
||||||
- `ResourceAccessProvider`
|
|
||||||
- lets modules register resource ACL behavior without importing each other.
|
|
||||||
- `SecretProvider`
|
|
||||||
- encrypts/decrypts named secrets without tying consumers to access models.
|
|
||||||
- `AuditSink`
|
|
||||||
- records audit events without importing audit storage models.
|
|
||||||
|
|
||||||
DTOs should be small and serializable:
|
|
||||||
|
|
||||||
- `PrincipalRef`
|
|
||||||
- `AccountRef`
|
|
||||||
- `TenantRef`
|
|
||||||
- `UserRef`
|
|
||||||
- `GroupRef`
|
|
||||||
- `RoleRef`
|
|
||||||
|
|
||||||
## Extraction Stages
|
|
||||||
|
|
||||||
### Stage 0: Baseline Safeguards
|
|
||||||
|
|
||||||
Status: started.
|
|
||||||
|
|
||||||
- Document the kernel/platform module model.
|
|
||||||
- Add dependency-boundary checks.
|
|
||||||
- Add backend module permutation startup tests.
|
|
||||||
- Inventory access/auth/RBAC imports.
|
|
||||||
- Draft this extraction plan.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- `scripts/check_dependency_boundaries.py` passes.
|
|
||||||
- backend module permutation tests start every supported module combination.
|
|
||||||
- current direct imports are tracked as explicit transitional debt.
|
|
||||||
|
|
||||||
### Stage 1: Contract Definitions
|
|
||||||
|
|
||||||
Add kernel-owned protocols and DTOs for access-related interaction.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Add protocol definitions under a kernel contract package.
|
|
||||||
- Add registry/capability names for access services.
|
|
||||||
- Keep existing core dependency functions as compatibility wrappers.
|
|
||||||
- Make wrappers resolve through capabilities when `govoplan-access` is present.
|
|
||||||
- Add tests for missing-capability behavior and clear error messages.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Feature modules can receive principal/tenant/group/user references without
|
|
||||||
importing access ORM models.
|
|
||||||
- Existing routes continue to work through compatibility wrappers.
|
|
||||||
|
|
||||||
### Stage 2: Create `govoplan-access`
|
|
||||||
|
|
||||||
Create the new repository/package and move the access seed package into it.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Create package metadata and entry points for `govoplan-access`.
|
|
||||||
- Move `govoplan_core/access` implementation into the new package.
|
|
||||||
- Publish `ModuleManifest(id="access")`.
|
|
||||||
- Register access permissions, role templates, routers, and migrations.
|
|
||||||
- Add compatibility imports in core where needed.
|
|
||||||
- Add release/dev dependency entries in core.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- `govoplan-core + govoplan-access` starts through normal module discovery.
|
|
||||||
- `govoplan-core` compatibility imports still work during transition.
|
|
||||||
- access manifest, migrations, and route contributions are discovered by core.
|
|
||||||
|
|
||||||
### Stage 3: Move Live Models And Migrations
|
|
||||||
|
|
||||||
Move active identity/access models out of `govoplan_core.db.models`.
|
|
||||||
|
|
||||||
Current state: the stable table naming strategy is to keep legacy table names
|
|
||||||
and move SQLAlchemy class definitions under their platform owners. The old
|
|
||||||
`govoplan_core.db.models` compatibility re-export has been removed; callers
|
|
||||||
must import module-owned models or use kernel capabilities. The earlier
|
|
||||||
`access_*` candidate tables are not active metadata.
|
|
||||||
|
|
||||||
Current table ownership:
|
|
||||||
|
|
||||||
- `govoplan-tenancy`: `tenants`
|
|
||||||
- `govoplan-access`: `accounts`, tenant memberships in `users`, `groups`,
|
|
||||||
`roles`, role/group assignment tables, `api_keys`, and `auth_sessions`
|
|
||||||
- `govoplan-admin`: governance templates and assignments
|
|
||||||
- `govoplan-audit`: audit log
|
|
||||||
- `govoplan-core`: system settings
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- [x] Decide the stable table naming strategy.
|
|
||||||
- [x] Map legacy model names to access-owned model classes.
|
|
||||||
- [x] Keep database table names where possible to avoid data migration churn.
|
|
||||||
- [x] Add Alembic migration metadata owned by the platform module owners.
|
|
||||||
- [x] Remove core model compatibility aliases after callers moved to
|
|
||||||
module-owned imports.
|
|
||||||
- [x] Update bootstrap/create-all compatibility paths.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Existing development databases migrate without data loss.
|
|
||||||
- New databases initialize with module-owned metadata.
|
|
||||||
- Core no longer owns or re-exports live account/user/group/role/session/API-key
|
|
||||||
model definitions.
|
|
||||||
|
|
||||||
### Stage 4: Move Auth Routes And Dependencies
|
|
||||||
|
|
||||||
Move authentication, sessions, API keys, and principal dependencies.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Move `/api/v1/auth/*` implementation to access.
|
|
||||||
- Move session/API-key services to access.
|
|
||||||
- Keep core route compatibility only if required by clients.
|
|
||||||
- Replace feature-module imports of core auth dependencies with the
|
|
||||||
access-published FastAPI dependency API.
|
|
||||||
- Add tests for login, session refresh, tenant selection, API-key auth, and
|
|
||||||
missing access capability failures.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Auth behavior works through the access module.
|
|
||||||
- Feature modules do not import access internals except the published
|
|
||||||
`govoplan_access.backend.auth.dependencies` dependency API used by FastAPI
|
|
||||||
routers.
|
|
||||||
- Core can explain startup failure clearly if auth-required routes are enabled
|
|
||||||
without the access capability.
|
|
||||||
|
|
||||||
### Stage 5: Move Admin And WebUI Contributions
|
|
||||||
|
|
||||||
Move access administration UI/API ownership into modules.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- [x] Move users, groups, roles, API keys, and access settings pages into
|
|
||||||
`govoplan-access`.
|
|
||||||
- [ ] Move tenant-specific pages to `govoplan-tenancy` when that module exists, or
|
|
||||||
keep them temporarily in access with clear boundaries.
|
|
||||||
- [x] Move overview, system settings, and governance-template panels into
|
|
||||||
`govoplan-admin`.
|
|
||||||
- [x] Register admin navigation and admin sections through module
|
|
||||||
contributions.
|
|
||||||
- [x] Keep core shell, layout, route rendering, and generic components only.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Core WebUI shell renders admin/access pages from module contributions.
|
|
||||||
- Core does not import access page components directly.
|
|
||||||
- Module nav and route metadata remain serializable.
|
|
||||||
- The access admin shell consumes module-owned `admin.sections` capability
|
|
||||||
contributions without importing sibling module panels.
|
|
||||||
|
|
||||||
### Stage 6: Decouple Feature Modules
|
|
||||||
|
|
||||||
Remove direct ORM/model imports from files, mail, and campaign.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Replace `Group`, `Tenant`, `User`, and `UserGroupMembership` imports with
|
|
||||||
directory/capability lookups.
|
|
||||||
- Replace direct cross-module cleanup/count queries with registered providers,
|
|
||||||
events, or module-owned API contracts.
|
|
||||||
- Replace mail-profile ownership resolution with stable owner references.
|
|
||||||
- Replace campaign/file access checks with resource ACL provider contracts.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` do not import
|
|
||||||
access implementation modules or sibling feature modules.
|
|
||||||
- Dependency-boundary allowlist stays empty after each replacement.
|
|
||||||
|
|
||||||
### Stage 7: Remove Compatibility Debt
|
|
||||||
|
|
||||||
Finalize the split.
|
|
||||||
|
|
||||||
Tasks:
|
|
||||||
|
|
||||||
- Remove obsolete core compatibility aliases.
|
|
||||||
- Keep the dependency-boundary checker allowlist empty.
|
|
||||||
- Update release dependency docs.
|
|
||||||
- Update operator migration notes.
|
|
||||||
- Add full module permutation tests including access-present and access-absent
|
|
||||||
behavior.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Core is a composition kernel.
|
|
||||||
- Access behavior is owned by `govoplan-access`.
|
|
||||||
- Feature modules communicate through kernel contracts, capabilities, events,
|
|
||||||
and their own APIs.
|
|
||||||
|
|
||||||
## Immediate Backlog
|
|
||||||
|
|
||||||
- [x] Document kernel/platform module model.
|
|
||||||
- [x] Add dependency-boundary check.
|
|
||||||
- [x] Add backend permutation startup smoke checks.
|
|
||||||
- [x] Inventory access/auth/RBAC import debt.
|
|
||||||
- [x] Draft access extraction plan.
|
|
||||||
- [x] Define kernel access DTOs and protocols.
|
|
||||||
- [x] Add access capability registry names.
|
|
||||||
- [x] Register live access directory and tenant resolver capability
|
|
||||||
implementations backed by the current compatibility tables.
|
|
||||||
- [x] Register an access tenant-provisioner capability so tenancy can seed
|
|
||||||
access-owned default roles without importing access service internals.
|
|
||||||
- [x] Replace `govoplan-files` direct user/group/tenant model imports with the
|
|
||||||
`access.directory` capability for group membership and share-target checks.
|
|
||||||
- [x] Replace `govoplan-files` direct campaign model imports with the
|
|
||||||
`campaigns.access` capability for campaign share/existence checks.
|
|
||||||
- [x] Replace `govoplan-mail` direct user/group/tenant model imports with
|
|
||||||
mail-owned policy storage plus access-directory validation and legacy-read
|
|
||||||
fallback.
|
|
||||||
- [x] Replace `govoplan-mail` direct campaign model imports with the
|
|
||||||
`campaigns.mailPolicyContext` capability for campaign-scoped mail policy and
|
|
||||||
owner context.
|
|
||||||
- [x] Replace `govoplan-campaign` runtime user/group/tenant model imports with
|
|
||||||
access-directory lookups and string-based ORM relationships.
|
|
||||||
- [x] Create `govoplan-access` repository workspace and issue workflow scaffold.
|
|
||||||
- [x] Create `govoplan-access` repository/package skeleton.
|
|
||||||
- [x] Move `govoplan_core/access` seed package into `govoplan-access`.
|
|
||||||
- [x] Add compatibility wrappers in core for old import paths.
|
|
||||||
- [x] Route existing auth dependency wrappers through access principal/evaluator capabilities.
|
|
||||||
- [x] Move FastAPI auth dependency wrappers out of core and into
|
|
||||||
`govoplan-access`.
|
|
||||||
- [x] Move session, API-key, and password helper services into `govoplan-access`.
|
|
||||||
- [x] Move interactive auth/session routes behind access module manifest.
|
|
||||||
- [x] Move legacy admin/API-key routes behind access module manifest.
|
|
||||||
- [x] Move access-owned legacy admin service helpers into `govoplan-access`.
|
|
||||||
- [x] Move governance-template CRUD helpers and routes into `govoplan-admin`.
|
|
||||||
- [x] Move governance-template materialization of access-owned groups and roles
|
|
||||||
behind the `access.governanceMaterializer` capability.
|
|
||||||
- [x] Replace legacy access-to-files/campaign admin lookups with module
|
|
||||||
tenant-summary and group-delete veto providers.
|
|
||||||
- [x] Split legacy admin route contribution across `govoplan-admin`,
|
|
||||||
`govoplan-tenancy`, `govoplan-policy`, `govoplan-audit`, and
|
|
||||||
`govoplan-access` route slices.
|
|
||||||
- [x] Move legacy admin route-handler ownership out of the access compatibility
|
|
||||||
router into access, admin, tenancy, policy, and audit module routers.
|
|
||||||
- [x] Move shared system-settings, tenant-governance, slug/error, and
|
|
||||||
tenant-count helpers out of access internals into core settings/tenancy
|
|
||||||
helpers used by the split platform route modules.
|
|
||||||
- [x] Move campaign schema routes into the campaign route contribution.
|
|
||||||
- [x] Move development mailbox routes into the mail route contribution.
|
|
||||||
- [x] Replace core Celery direct campaign/mail imports with the
|
|
||||||
`campaigns.deliveryTasks` capability.
|
|
||||||
- [x] Replace core retention direct campaign queries with
|
|
||||||
`campaigns.policyContext` and `campaigns.retention` capabilities.
|
|
||||||
- [x] Replace core `create_all` feature-model imports with module registry
|
|
||||||
metadata discovery.
|
|
||||||
- [x] Remove transitional boundary checker allowlist entries.
|
|
||||||
- [x] Move live legacy model definitions out of core and into
|
|
||||||
`govoplan-access` while preserving existing table names.
|
|
||||||
- [x] Split the transitional access-owned legacy model graph further into
|
|
||||||
tenancy, audit, admin, access, and core settings ownership.
|
|
||||||
- [x] Reverse the tenancy/access dependency direction so `govoplan-access`
|
|
||||||
depends on `govoplan-tenancy`, and the registry inserts tenancy before access.
|
|
||||||
- [x] Replace tenancy-to-access default role seeding with an access
|
|
||||||
tenant-provisioner capability.
|
|
||||||
- [x] Replace tenancy-to-access owner candidate and owner membership handling
|
|
||||||
with the access tenant-provisioner capability.
|
|
||||||
- [x] Replace admin overview counts and audit actor lookup/filtering with the
|
|
||||||
`access.administration` capability.
|
|
||||||
- [x] Replace feature-module direct user/group/tenant model imports.
|
|
||||||
- [x] `govoplan-files`
|
|
||||||
- [x] `govoplan-mail`
|
|
||||||
- [x] `govoplan-campaign`
|
|
||||||
- [x] Replace files/mail/campaign direct cross-module SQL lookups with
|
|
||||||
provider/capability contracts.
|
|
||||||
- [x] Move access admin WebUI pages to module route contributions.
|
|
||||||
- [x] Move generic admin-owned WebUI panels into `govoplan-admin` and register
|
|
||||||
them through the `admin.sections` UI capability.
|
|
||||||
- [x] Remove transitional boundary checker allowlist entries as each contract
|
|
||||||
lands.
|
|
||||||
- [x] Remove old core import shims for access models, access routes, admin
|
|
||||||
service helpers, and access-owned security services.
|
|
||||||
- [ ] Move campaign-scoped mail policy ownership fully behind an API/event
|
|
||||||
workflow if direct synchronous capability calls become too tight for later
|
|
||||||
deployment boundaries.
|
|
||||||
|
|
||||||
## Open Decisions
|
|
||||||
|
|
||||||
- Is `govoplan-access` required for any authenticated deployment, while
|
|
||||||
core-only remains a diagnostics/settings shell?
|
|
||||||
- Tenant models live in `govoplan-tenancy`; `govoplan-access` depends on
|
|
||||||
tenancy for authenticated platform composition.
|
|
||||||
- Historical table names remain stable for painless migrations.
|
|
||||||
- Should secret encryption remain a kernel primitive or become an access-owned
|
|
||||||
capability?
|
|
||||||
- Audit log storage lives in `govoplan-audit`; policy provenance can build on
|
|
||||||
that module boundary.
|
|
||||||
- How long should core keep compatibility route paths for existing clients?
|
|
||||||
|
|
||||||
## Verification
|
|
||||||
|
|
||||||
Use the following checks while extracting access:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
|
||||||
```
|
|
||||||
|
|
||||||
For each removed dependency-boundary exception, add or update a focused test
|
|
||||||
that proves the replacement contract starts without the old direct import.
|
|
||||||
288
docs/ACCESS_RBAC_MODEL.md
Normal file
288
docs/ACCESS_RBAC_MODEL.md
Normal file
@@ -0,0 +1,288 @@
|
|||||||
|
# GovOPlaN RBAC And Resource-Access Model
|
||||||
|
|
||||||
|
**Updated:** 2026-07-09
|
||||||
|
|
||||||
|
## Authorization Equation
|
||||||
|
|
||||||
|
An operation is permitted only when every applicable layer allows it:
|
||||||
|
|
||||||
|
```text
|
||||||
|
effective role/API-key capability
|
||||||
|
AND resource ownership/share access
|
||||||
|
AND workflow state
|
||||||
|
AND active governance/policy constraints
|
||||||
|
```
|
||||||
|
|
||||||
|
RBAC answers what an actor may do. ACLs answer which resource the actor may do
|
||||||
|
it to. Workflow state and policy decide whether the operation is currently
|
||||||
|
valid.
|
||||||
|
|
||||||
|
## Identity And Scope
|
||||||
|
|
||||||
|
```text
|
||||||
|
Account global login identity
|
||||||
|
+- User membership tenant-local identity
|
||||||
|
+- direct tenant roles
|
||||||
|
+- active group memberships
|
||||||
|
| +- inherited tenant roles
|
||||||
|
+- tenant-local API keys
|
||||||
|
|
||||||
|
Account
|
||||||
|
+- direct system-role assignments
|
||||||
|
```
|
||||||
|
|
||||||
|
A browser session has one active tenant membership. System privileges do not
|
||||||
|
silently grant tenant data access. API keys remain tenant-local and receive the
|
||||||
|
intersection of their configured scopes and their owner's live tenant scopes on
|
||||||
|
every request.
|
||||||
|
|
||||||
|
## Wildcards
|
||||||
|
|
||||||
|
```text
|
||||||
|
tenant:* every canonical tenant permission
|
||||||
|
system:* every canonical system permission
|
||||||
|
* legacy alias interpreted as tenant:* only
|
||||||
|
```
|
||||||
|
|
||||||
|
Tenant wildcards never grant system permissions.
|
||||||
|
|
||||||
|
## Canonical Tenant Permissions
|
||||||
|
|
||||||
|
Campaigns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
campaign:read
|
||||||
|
campaign:create
|
||||||
|
campaign:update
|
||||||
|
campaign:copy
|
||||||
|
campaign:archive
|
||||||
|
campaign:delete
|
||||||
|
campaign:share
|
||||||
|
campaign:validate
|
||||||
|
campaign:build
|
||||||
|
campaign:review
|
||||||
|
campaign:send_test
|
||||||
|
campaign:queue
|
||||||
|
campaign:control
|
||||||
|
campaign:send
|
||||||
|
campaign:retry
|
||||||
|
campaign:reconcile
|
||||||
|
```
|
||||||
|
|
||||||
|
Recipients:
|
||||||
|
|
||||||
|
```text
|
||||||
|
recipients:read
|
||||||
|
recipients:write
|
||||||
|
recipients:import
|
||||||
|
recipients:export
|
||||||
|
```
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
```text
|
||||||
|
files:read
|
||||||
|
files:download
|
||||||
|
files:upload
|
||||||
|
files:organize
|
||||||
|
files:share
|
||||||
|
files:delete
|
||||||
|
files:admin
|
||||||
|
```
|
||||||
|
|
||||||
|
Reports and audit:
|
||||||
|
|
||||||
|
```text
|
||||||
|
reports:read
|
||||||
|
reports:export
|
||||||
|
reports:send
|
||||||
|
audit:read
|
||||||
|
```
|
||||||
|
|
||||||
|
Mail servers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
mail_servers:read
|
||||||
|
mail_servers:use
|
||||||
|
mail_servers:test
|
||||||
|
mail_servers:write
|
||||||
|
mail_servers:manage_credentials
|
||||||
|
```
|
||||||
|
|
||||||
|
Tenant administration:
|
||||||
|
|
||||||
|
```text
|
||||||
|
admin:users:read
|
||||||
|
admin:users:create
|
||||||
|
admin:users:update
|
||||||
|
admin:users:suspend
|
||||||
|
|
||||||
|
admin:groups:read
|
||||||
|
admin:groups:write
|
||||||
|
admin:groups:manage_members
|
||||||
|
|
||||||
|
admin:roles:read
|
||||||
|
admin:roles:write
|
||||||
|
admin:roles:assign
|
||||||
|
|
||||||
|
admin:api_keys:read
|
||||||
|
admin:api_keys:create
|
||||||
|
admin:api_keys:revoke
|
||||||
|
|
||||||
|
admin:settings:read
|
||||||
|
admin:settings:write
|
||||||
|
admin:policies:read
|
||||||
|
admin:policies:write
|
||||||
|
```
|
||||||
|
|
||||||
|
## Canonical System Permissions
|
||||||
|
|
||||||
|
```text
|
||||||
|
system:tenants:read
|
||||||
|
system:tenants:create
|
||||||
|
system:tenants:update
|
||||||
|
system:tenants:suspend
|
||||||
|
|
||||||
|
system:accounts:read
|
||||||
|
system:accounts:create
|
||||||
|
system:accounts:update
|
||||||
|
system:accounts:suspend
|
||||||
|
|
||||||
|
system:roles:read
|
||||||
|
system:roles:write
|
||||||
|
system:roles:assign
|
||||||
|
|
||||||
|
system:access:read
|
||||||
|
system:access:assign
|
||||||
|
|
||||||
|
system:audit:read
|
||||||
|
system:settings:read
|
||||||
|
system:settings:write
|
||||||
|
system:governance:read
|
||||||
|
system:governance:write
|
||||||
|
```
|
||||||
|
|
||||||
|
`system:access:*` remains a read/assignment boundary for cross-tenant and
|
||||||
|
system access handling. It is not a separate primary UI area.
|
||||||
|
|
||||||
|
## Default Tenant Roles
|
||||||
|
|
||||||
|
- **Owner:** `tenant:*`. At least one active operational owner must remain.
|
||||||
|
- **Tenant administrator:** settings, policies, users, groups, roles, API keys,
|
||||||
|
and read access to campaigns/files/reports/audit. Real delivery remains
|
||||||
|
separately delegable.
|
||||||
|
- **Administrator:** all tenant permissions for upgraded installations.
|
||||||
|
- **Access administrator:** membership and assignment management within
|
||||||
|
delegation limits.
|
||||||
|
- **Campaign manager:** prepare, validate, and build campaigns; no review
|
||||||
|
approval or real delivery by default.
|
||||||
|
- **Reviewer:** inspect and approve prepared campaign messages.
|
||||||
|
- **Sender:** mock-test, queue, control, send, retry, and reconcile prepared
|
||||||
|
campaigns; can use/test approved mail profiles.
|
||||||
|
- **File manager:** managed file operations without campaign delivery rights.
|
||||||
|
- **Viewer:** read campaigns, recipients, files, and reports.
|
||||||
|
- **Auditor:** read campaigns, recipient evidence, reports, and audit records;
|
||||||
|
export detailed evidence.
|
||||||
|
|
||||||
|
## Default System Roles
|
||||||
|
|
||||||
|
- **System owner:** `system:*`, protected. At least one active account must
|
||||||
|
retain it.
|
||||||
|
- **System administrator:** all specific system permissions, editable and not
|
||||||
|
protected.
|
||||||
|
- **System auditor:** read-only system registry/settings/governance/audit role,
|
||||||
|
editable.
|
||||||
|
|
||||||
|
## Delegation Ceiling
|
||||||
|
|
||||||
|
For role definition, assignment, and API-key creation:
|
||||||
|
|
||||||
|
```text
|
||||||
|
requested scopes subset of actor delegateable scopes
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
1. Tenant roles may contain tenant scopes only.
|
||||||
|
2. System roles may contain system scopes only.
|
||||||
|
3. Definition rights and assignment rights are separate.
|
||||||
|
4. Group definition and group membership management are separate.
|
||||||
|
5. API-key scopes are intersected with the owner's current effective scopes on
|
||||||
|
every request.
|
||||||
|
6. Suspended accounts, users, tenants, or groups stop contributing access
|
||||||
|
immediately.
|
||||||
|
7. Administrative updates are field-sensitive; a user with only status
|
||||||
|
authority cannot change role assignments.
|
||||||
|
|
||||||
|
## Campaign Ownership And ACLs
|
||||||
|
|
||||||
|
A campaign has exactly one owner:
|
||||||
|
|
||||||
|
```text
|
||||||
|
owner user OR owner group
|
||||||
|
```
|
||||||
|
|
||||||
|
Additional active shares may target users or groups with `read` or `write`.
|
||||||
|
|
||||||
|
Resolution:
|
||||||
|
|
||||||
|
- owner user: read and write;
|
||||||
|
- member of owner group: read and write;
|
||||||
|
- explicit read share: read;
|
||||||
|
- explicit write share: read and write;
|
||||||
|
- `tenant:*`: tenant-wide ACL bypass;
|
||||||
|
- ordinary campaign permission without ownership/share: no object access.
|
||||||
|
|
||||||
|
ACLs do not add capabilities. A write share still needs the specific permission
|
||||||
|
for update, validation, review, send, report, retry, or reconciliation.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
The current file access model distinguishes tenant-level file capabilities from
|
||||||
|
space/folder/file ownership:
|
||||||
|
|
||||||
|
| Scope | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `files:read` | browse/read visible file spaces and metadata |
|
||||||
|
| `files:download` | download file content when ACL permits |
|
||||||
|
| `files:upload` | create files in writable spaces |
|
||||||
|
| `files:organize` | create folders, move files, and update metadata where ACL permits |
|
||||||
|
| `files:share` | share files/spaces according to owner and policy rules |
|
||||||
|
| `files:delete` | delete or retire files where ACL permits |
|
||||||
|
| `files:admin` | tenant-wide administration of user/group file spaces |
|
||||||
|
|
||||||
|
External file connections and spaces are additionally constrained by connector
|
||||||
|
policy and owner/group assignment in the files module.
|
||||||
|
|
||||||
|
## Mail Servers
|
||||||
|
|
||||||
|
| Scope | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `mail_servers:read` | profile metadata and effective policy visibility |
|
||||||
|
| `mail_servers:use` | use an allowed profile for a campaign or message flow |
|
||||||
|
| `mail_servers:test` | run connectivity tests without revealing secrets |
|
||||||
|
| `mail_servers:write` | create/update profile metadata where policy allows |
|
||||||
|
| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or lower-level credentials where policy allows |
|
||||||
|
|
||||||
|
Reusable encrypted profiles exist. Effective usability is also constrained by
|
||||||
|
hierarchical mail-profile policy, ownership, allowed/forced profile sets,
|
||||||
|
credential inheritance mode, lower-level override switches, and allow/deny
|
||||||
|
patterns.
|
||||||
|
|
||||||
|
## Compatibility Aliases
|
||||||
|
|
||||||
|
Compatibility aliases may exist in backend code for upgraded installations, but
|
||||||
|
new UI and docs should use canonical scopes.
|
||||||
|
|
||||||
|
Current alias direction:
|
||||||
|
|
||||||
|
```text
|
||||||
|
* -> tenant:*
|
||||||
|
system:tenants:write -> create/update/suspend tenant scopes
|
||||||
|
system:access:write -> system access assignment/write scopes
|
||||||
|
```
|
||||||
|
|
||||||
|
A separate `retention:*` family is not currently canonical because retention is
|
||||||
|
managed through system settings and tenant policy scopes. Add it only if
|
||||||
|
retention operation duties need separation from general policy/settings
|
||||||
|
administration.
|
||||||
121
docs/ACTION_EFFECT_AUTOMATION_LAYER.md
Normal file
121
docs/ACTION_EFFECT_AUTOMATION_LAYER.md
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
# Action, Effect, And Automation Layer
|
||||||
|
|
||||||
|
GovOPlaN needs an automation layer because administrative processes will not be
|
||||||
|
only linear screen flows. Workflows, schedules, imports, connectors, policies,
|
||||||
|
and external events all need to request governed actions without bypassing the
|
||||||
|
same safety rules that apply to human users.
|
||||||
|
|
||||||
|
The first implementation should live in `govoplan-workflow` and core contracts.
|
||||||
|
Create a separate `govoplan-automation` module only if action planning,
|
||||||
|
schedulers, rule execution, or cross-module automation become too broad for
|
||||||
|
workflow ownership.
|
||||||
|
|
||||||
|
## Layer Purpose
|
||||||
|
|
||||||
|
The automation layer should provide:
|
||||||
|
|
||||||
|
- a typed action catalogue
|
||||||
|
- a typed effect catalogue
|
||||||
|
- consequence preview before execution
|
||||||
|
- policy and permission checks
|
||||||
|
- idempotent execution
|
||||||
|
- audit and provenance records
|
||||||
|
- retry, quarantine, and manual exception handling
|
||||||
|
- system-actor execution without hiding responsibility
|
||||||
|
|
||||||
|
Automation is not a shortcut around module boundaries. It is a governed caller
|
||||||
|
of module capabilities.
|
||||||
|
|
||||||
|
## Action Definition
|
||||||
|
|
||||||
|
An `ActionDefinition` describes something a human or system actor can request.
|
||||||
|
|
||||||
|
Recommended fields:
|
||||||
|
|
||||||
|
- `action_key`
|
||||||
|
- owning module
|
||||||
|
- input schema
|
||||||
|
- actor requirements and required scopes
|
||||||
|
- required capabilities
|
||||||
|
- policy checks
|
||||||
|
- risk level
|
||||||
|
- reversibility class: reversible, compensatable, corrective-only, or
|
||||||
|
irreversible
|
||||||
|
- expected effects
|
||||||
|
- idempotency key strategy
|
||||||
|
- audit event names
|
||||||
|
- preview provider
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- create a case from a form submission
|
||||||
|
- assign a task
|
||||||
|
- generate a document from a template
|
||||||
|
- send a postbox message
|
||||||
|
- send an email notification
|
||||||
|
- append evidence to records
|
||||||
|
- create a payment request
|
||||||
|
- call an external connector
|
||||||
|
|
||||||
|
## Effect Definition
|
||||||
|
|
||||||
|
An `EffectDefinition` describes the expected and observed result of an action.
|
||||||
|
|
||||||
|
Recommended fields:
|
||||||
|
|
||||||
|
- `effect_key`
|
||||||
|
- affected module and resource references
|
||||||
|
- external system references where applicable
|
||||||
|
- created, changed, deleted, sent, notified, locked, or retained markers
|
||||||
|
- visibility and privacy classification
|
||||||
|
- audit event references
|
||||||
|
- rollback or compensation hints
|
||||||
|
- operator-facing explanation
|
||||||
|
|
||||||
|
Effects should be recorded even when execution fails partially. This makes
|
||||||
|
manual recovery and audit review possible.
|
||||||
|
|
||||||
|
## Execution Model
|
||||||
|
|
||||||
|
The runner should execute an action plan as follows:
|
||||||
|
|
||||||
|
1. Resolve actor context: human, delegated actor, or system actor.
|
||||||
|
2. Validate input schema.
|
||||||
|
3. Resolve required module capabilities.
|
||||||
|
4. Run permission and policy checks.
|
||||||
|
5. Generate a consequence preview.
|
||||||
|
6. Reserve or verify the idempotency key.
|
||||||
|
7. Execute the owning module capability.
|
||||||
|
8. Record observed effects.
|
||||||
|
9. Emit events and audit records.
|
||||||
|
10. Mark the command complete, retryable, quarantined, or requiring manual
|
||||||
|
intervention.
|
||||||
|
|
||||||
|
The runner must never advance workflow state past a required side effect unless
|
||||||
|
the action definition explicitly allows asynchronous completion and the pending
|
||||||
|
state is visible.
|
||||||
|
|
||||||
|
## Failure States
|
||||||
|
|
||||||
|
Automation should use explicit failure states:
|
||||||
|
|
||||||
|
- `blocked`: policy, permission, missing capability, or invalid input prevents
|
||||||
|
execution.
|
||||||
|
- `retryable`: transient transport, timeout, rate-limit, or lock conflict.
|
||||||
|
- `quarantined`: unexpected response, schema mismatch, unsafe partial result,
|
||||||
|
or unknown external state.
|
||||||
|
- `manual_required`: human decision or correction is needed.
|
||||||
|
- `compensation_required`: a later action must correct an already observed
|
||||||
|
side effect.
|
||||||
|
|
||||||
|
These states should be visible in workflow, task, and admin diagnostics.
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Core may own stable DTOs, registry contracts, and generic audit/event hooks.
|
||||||
|
`govoplan-workflow` should own the first runner because workflow is the first
|
||||||
|
module that coordinates cross-module process actions.
|
||||||
|
|
||||||
|
Domain modules own their own action providers. For example, templates own
|
||||||
|
document generation actions, postbox owns postbox message actions, and
|
||||||
|
connectors own external handoff actions.
|
||||||
@@ -1,185 +0,0 @@
|
|||||||
# Module Catalog Trust And Licensing
|
|
||||||
|
|
||||||
GovOPlaN module install and uninstall must remain operator-controlled. The
|
|
||||||
running server may plan and validate package changes, but package mutation is
|
|
||||||
performed by the separate installer daemon or an operator shell during
|
|
||||||
maintenance mode.
|
|
||||||
|
|
||||||
## Roles
|
|
||||||
|
|
||||||
`govoplan-web` is the public static distribution surface for official catalog
|
|
||||||
resources:
|
|
||||||
|
|
||||||
- signed module package catalogs, grouped by release channel
|
|
||||||
- public catalog keyrings
|
|
||||||
- public license verification keyrings
|
|
||||||
- examples and operator-facing download paths
|
|
||||||
|
|
||||||
`govoplan-core` is the verifier and orchestrator:
|
|
||||||
|
|
||||||
- fetches a local or remote module catalog
|
|
||||||
- verifies catalog signatures against configured trusted keys
|
|
||||||
- enforces approved release channels
|
|
||||||
- rejects expired or not-yet-valid catalogs
|
|
||||||
- records accepted catalog sequence numbers for replay protection
|
|
||||||
- checks catalog entry license feature requirements before planning installs
|
|
||||||
- writes installer plans and request records
|
|
||||||
|
|
||||||
Feature and platform modules own their package artifacts, manifests, migration
|
|
||||||
metadata, retirement providers, and optional lifecycle behavior.
|
|
||||||
|
|
||||||
## Catalog Source
|
|
||||||
|
|
||||||
Core accepts either a local catalog file or a remote URL:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
|
||||||
```
|
|
||||||
|
|
||||||
If both file and URL are set, the URL wins. The cache is used when a remote
|
|
||||||
fetch fails, so an operator can still inspect the last known catalog. A cached
|
|
||||||
catalog must still pass signature, freshness, channel, and replay validation.
|
|
||||||
|
|
||||||
## Catalog Shape
|
|
||||||
|
|
||||||
An official catalog is a JSON object with:
|
|
||||||
|
|
||||||
- `catalog_version`
|
|
||||||
- `channel`
|
|
||||||
- `sequence`
|
|
||||||
- `generated_at`
|
|
||||||
- `not_before` when delayed activation is needed
|
|
||||||
- `expires_at`
|
|
||||||
- `modules`
|
|
||||||
- `signatures`
|
|
||||||
|
|
||||||
Each module entry can declare:
|
|
||||||
|
|
||||||
- backend package name and pinned install reference
|
|
||||||
- WebUI package name and pinned install reference
|
|
||||||
- display metadata and tags
|
|
||||||
- `license_features`, the feature entitlements required to plan that install
|
|
||||||
|
|
||||||
The signature is Ed25519 over canonical JSON with both `signature` and
|
|
||||||
`signatures` removed. Core accepts the legacy single `signature` field and the
|
|
||||||
new `signatures` array.
|
|
||||||
|
|
||||||
## Keyrings And Rotation
|
|
||||||
|
|
||||||
Trusted catalog keys are configured locally:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":"<base64 public key>"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
For development or tightly controlled deployments, a keyring can be read from a
|
|
||||||
URL and cached:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Production installations should pin the trusted keyring locally or ship it
|
|
||||||
through deployment configuration. Fetching trusted keys from the same public
|
|
||||||
origin as the catalog is convenient, but that origin must not become the only
|
|
||||||
trust root.
|
|
||||||
|
|
||||||
Keyring entries support:
|
|
||||||
|
|
||||||
- `key_id`
|
|
||||||
- `public_key` or `public_key_base64`
|
|
||||||
- `status`: `active`, `next`, `retired`, `revoked`, or `disabled`
|
|
||||||
- `not_before`
|
|
||||||
- `not_after`
|
|
||||||
|
|
||||||
Rotation process:
|
|
||||||
|
|
||||||
1. Add the next public key to the local trusted keyring with status `next`.
|
|
||||||
2. Publish catalogs signed by both current and next keys.
|
|
||||||
3. Upgrade installations so the next key is locally trusted.
|
|
||||||
4. Promote the next key to `active`.
|
|
||||||
5. Retire the old key only after every supported installation trusts the new
|
|
||||||
key.
|
|
||||||
6. Mark a compromised key `revoked` and publish a higher sequence catalog
|
|
||||||
signed by an uncompromised key.
|
|
||||||
|
|
||||||
## Replay And Freshness
|
|
||||||
|
|
||||||
Use replay state in production:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
Core records the accepted sequence per channel after a catalog entry is planned
|
|
||||||
from the admin interface. With strict sequence enforcement, a previously
|
|
||||||
accepted sequence is rejected; without strict enforcement, only older sequences
|
|
||||||
are rejected.
|
|
||||||
|
|
||||||
Catalogs should always expire. Long-lived catalogs make rollback and key
|
|
||||||
compromise harder to reason about.
|
|
||||||
|
|
||||||
## Release Channels
|
|
||||||
|
|
||||||
Approved channels are deployment policy:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
The admin UI can display other catalog metadata, but core rejects catalogs from
|
|
||||||
unapproved channels when validation is configured.
|
|
||||||
|
|
||||||
## Licensing
|
|
||||||
|
|
||||||
Catalog entries can require license features:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"license_features": ["module.mail", "support.standard"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Core checks those requirements against an offline license file before allowing
|
|
||||||
the entry into the install plan.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json
|
|
||||||
GOVOPLAN_LICENSE_ENFORCEMENT=true
|
|
||||||
GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json
|
|
||||||
```
|
|
||||||
|
|
||||||
License files are JSON objects with:
|
|
||||||
|
|
||||||
- `license_id`
|
|
||||||
- `subject`
|
|
||||||
- `features`
|
|
||||||
- `valid_from`
|
|
||||||
- `valid_until`
|
|
||||||
- `signature`
|
|
||||||
|
|
||||||
License enforcement can run in observe-only mode by leaving
|
|
||||||
`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license
|
|
||||||
data is surfaced as a warning but does not block planning.
|
|
||||||
|
|
||||||
Licensing is intentionally separate from open-source code licensing. The
|
|
||||||
catalog/license mechanism can govern support channels, official release
|
|
||||||
eligibility, hosted update access, professional support, or commercial
|
|
||||||
entitlements without changing the source license of the repositories.
|
|
||||||
|
|
||||||
## Production Gaps
|
|
||||||
|
|
||||||
The current implementation validates signed catalog metadata and offline
|
|
||||||
license entitlements. Production-grade distribution still needs:
|
|
||||||
|
|
||||||
- artifact digest verification for package archives or registry artifacts
|
|
||||||
- SBOM/provenance fields and verification workflow
|
|
||||||
- hardened catalog publishing pipeline in `govoplan-web`
|
|
||||||
- automated key rotation runbook and emergency revocation procedure
|
|
||||||
- license issuance and renewal tooling
|
|
||||||
- tests for remote catalog cache fallback and replay-state upgrades
|
|
||||||
|
|
||||||
@@ -27,6 +27,12 @@ trust_level = "trusted"
|
|||||||
[projects."/mnt/DATA/git/govoplan-access"]
|
[projects."/mnt/DATA/git/govoplan-access"]
|
||||||
trust_level = "trusted"
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
[projects."/mnt/DATA/git/govoplan-organizations"]
|
||||||
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
[projects."/mnt/DATA/git/govoplan-identity"]
|
||||||
|
trust_level = "trusted"
|
||||||
|
|
||||||
[projects."/mnt/DATA/git/govoplan-mail"]
|
[projects."/mnt/DATA/git/govoplan-mail"]
|
||||||
trust_level = "trusted"
|
trust_level = "trusted"
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,13 @@ Example: an application-handling package could configure a public portal form,
|
|||||||
a case workflow, task creation, a mail template, payment processing, access
|
a case workflow, task creation, a mail template, payment processing, access
|
||||||
roles, audit evidence, and the interface bindings between those modules.
|
roles, audit evidence, and the interface bindings between those modules.
|
||||||
|
|
||||||
|
The guiding reference scenario is the government-operations permit journey in
|
||||||
|
`docs/GOVOPLAN_MASTER_ROADMAP.md`: a person applies through the portal,
|
||||||
|
uploads files, receives workflow-driven messages and appointment proposals, has
|
||||||
|
a case opened, gets a permit generated from a template, and completes payment.
|
||||||
|
Configuration packages are the mechanism that should make such processes
|
||||||
|
reusable and safely importable.
|
||||||
|
|
||||||
This document is durable architecture context and should be mirrored to the
|
This document is durable architecture context and should be mirrored to the
|
||||||
Gitea wiki. Active implementation should be tracked in Gitea issues.
|
Gitea wiki. Active implementation should be tracked in Gitea issues.
|
||||||
|
|
||||||
@@ -97,6 +104,33 @@ export(selection, context) -> fragment, data_placeholders, warnings
|
|||||||
health(import_result, context) -> diagnostics
|
health(import_result, context) -> diagnostics
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The initial core contract lives in
|
||||||
|
`govoplan_core.core.configuration_packages`. Modules register providers through
|
||||||
|
module-specific capability keys and implement the `ConfigurationProvider`
|
||||||
|
protocol. The generic `configuration.provider` key names the contract and can be
|
||||||
|
used in package requirements; concrete providers such as `access.configuration`
|
||||||
|
are resolved by the admin package wizard. The first DTO surface includes package
|
||||||
|
manifests, module/capability requirements, module-owned fragments, diagnostics,
|
||||||
|
required operator data, dry-run plan items, apply results, export selections,
|
||||||
|
and export results.
|
||||||
|
|
||||||
|
The initial implementation includes provider-neutral orchestration helpers:
|
||||||
|
|
||||||
|
- `dry_run_configuration_package(...)`
|
||||||
|
- `apply_configuration_package(...)`
|
||||||
|
- `export_configuration_package(...)`
|
||||||
|
|
||||||
|
The first concrete provider is `govoplan_access.backend.configuration_provider`.
|
||||||
|
It supports access-owned `roles`, `groups`, and `group_role_assignments`
|
||||||
|
fragments and applies them idempotently.
|
||||||
|
|
||||||
|
The admin wizard backend starts with these routes:
|
||||||
|
|
||||||
|
- `GET /api/v1/admin/configuration-packages/catalog`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/dry-run`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/apply`
|
||||||
|
- `POST /api/v1/admin/configuration-packages/export`
|
||||||
|
|
||||||
## Import Flow
|
## Import Flow
|
||||||
|
|
||||||
1. Select a package from a trusted catalog or upload a package file.
|
1. Select a package from a trusted catalog or upload a package file.
|
||||||
@@ -168,6 +202,20 @@ Configuration catalogs should follow the existing module package catalog model:
|
|||||||
a file-backed or remotely fetched JSON catalog with Ed25519 signatures, channel
|
a file-backed or remotely fetched JSON catalog with Ed25519 signatures, channel
|
||||||
gating, trusted key ids, and operator-controlled trust policy.
|
gating, trusted key ids, and operator-controlled trust policy.
|
||||||
|
|
||||||
|
The initial catalog validator mirrors the module catalog environment model with
|
||||||
|
configuration-specific names:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG=/srv/govoplan/configuration-catalogs/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_URL=https://govoplan.example/configuration-catalogs/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/configuration-catalog-cache/stable.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/configuration-catalog-keyring.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/configuration-catalog-sequences.json
|
||||||
|
GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
Catalog entries should include:
|
Catalog entries should include:
|
||||||
|
|
||||||
- package id, version, name, description, publisher, tags, and channel
|
- package id, version, name, description, publisher, tags, and channel
|
||||||
|
|||||||
67
docs/DEPENDENCY_AUDITS.md
Normal file
67
docs/DEPENDENCY_AUDITS.md
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Dependency Audits
|
||||||
|
|
||||||
|
GovOPlaN keeps dependency vulnerability checks reproducible but separate from
|
||||||
|
the fast local smoke suite, because both Python and npm audits need network
|
||||||
|
metadata and can fail for newly disclosed advisories without a source change.
|
||||||
|
|
||||||
|
## Local Workflow
|
||||||
|
|
||||||
|
Install the development audit dependency once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m pip install -r requirements-dev.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Run both backend and WebUI production audits:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The script runs:
|
||||||
|
|
||||||
|
- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale
|
||||||
|
legacy editable package metadata, deprecated framework constants, and the
|
||||||
|
Starlette `TestClient` deprecation smoke when test dependencies are present
|
||||||
|
- `python -m pip_audit --progress-spinner off`
|
||||||
|
- `npm audit --omit=dev` in `webui`
|
||||||
|
|
||||||
|
For fast local checks without vulnerability metadata lookups, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This is also part of `scripts/check-focused.sh`, so resolver drift and
|
||||||
|
deprecation regressions fail close to the code change that introduced them.
|
||||||
|
|
||||||
|
Override tool paths when testing from a disposable environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHON=/tmp/govoplan-audit/bin/python \
|
||||||
|
NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## CI Workflow
|
||||||
|
|
||||||
|
`.gitea/workflows/dependency-audit.yml` installs release dependencies from
|
||||||
|
tagged package refs, installs `pip-audit`, and runs the same script on pushes,
|
||||||
|
pull requests, and a weekly schedule.
|
||||||
|
|
||||||
|
The workflow intentionally uses release dependency refs instead of local
|
||||||
|
`file:` or editable sibling paths. Development lockfiles may keep local module
|
||||||
|
links, but release audit results should represent the installable product.
|
||||||
|
|
||||||
|
## Recording Results
|
||||||
|
|
||||||
|
When closing or triaging dependency-audit issues, add a short dated note under
|
||||||
|
`docs/audits/`. Record:
|
||||||
|
|
||||||
|
- the commands that were run
|
||||||
|
- whether Python and npm passed
|
||||||
|
- any advisories accepted as temporary risk
|
||||||
|
- follow-up issue links for required upgrades
|
||||||
390
docs/DEPLOYMENT_OPERATOR_GUIDE.md
Normal file
390
docs/DEPLOYMENT_OPERATOR_GUIDE.md
Normal file
@@ -0,0 +1,390 @@
|
|||||||
|
# GovOPlaN Deployment Operator Guide
|
||||||
|
|
||||||
|
This guide defines the current install/runtime configuration contract and the
|
||||||
|
operator flow for a production-realistic self-hosted deployment. Keep secrets in
|
||||||
|
the deployment environment or a secret manager; do not commit populated `.env`
|
||||||
|
files.
|
||||||
|
|
||||||
|
## Runtime Configuration Contract
|
||||||
|
|
||||||
|
### Required Runtime Identity
|
||||||
|
|
||||||
|
| Setting | Required outside dev | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `APP_ENV` | yes | Runtime profile. Use `prod`, `staging`, or a deployment-specific value outside local development. |
|
||||||
|
| `MASTER_KEY_B64` | yes | Fernet key or base64 encoded 32-byte key used for encrypted module secrets. Rotate through an explicit operator plan. |
|
||||||
|
| `DATABASE_URL` | yes | SQLAlchemy database URL for core and installed modules. SQLite is supported for dev/small installs; PostgreSQL is the preferred production target. |
|
||||||
|
| `ENABLED_MODULES` | yes | Comma-separated startup module set. Keep `tenancy,access` enabled; keep `admin` enabled for operator UI. |
|
||||||
|
|
||||||
|
Generate a local key for a new non-production environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python - <<'PY'
|
||||||
|
from cryptography.fernet import Fernet
|
||||||
|
print(Fernet.generate_key().decode())
|
||||||
|
PY
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database And Migrations
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. |
|
||||||
|
| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. |
|
||||||
|
| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. |
|
||||||
|
|
||||||
|
Operator rule: take a database backup before applying migrations or destructive
|
||||||
|
module retirement. For non-SQLite databases, configure deployment-specific
|
||||||
|
backup/restore hooks for the module installer.
|
||||||
|
|
||||||
|
### PostgreSQL Production Target
|
||||||
|
|
||||||
|
PostgreSQL is the primary development and production target. SQLite remains
|
||||||
|
supported only for tiny disposable profiles and unit-test style smoke runs.
|
||||||
|
Production/staging deployments should use a managed PostgreSQL database and
|
||||||
|
explicit migration commands.
|
||||||
|
|
||||||
|
Install the server extra so the `psycopg` driver is available:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Example runtime database URLs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DATABASE_URL='postgresql+psycopg://govoplan:change-me@db.example.internal:5432/govoplan'
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS='postgresql://govoplan:change-me@db.example.internal:5432/govoplan'
|
||||||
|
```
|
||||||
|
|
||||||
|
Use the SQLAlchemy URL for GovOPlaN. Use the pg-tools URL for `pg_dump`,
|
||||||
|
`pg_restore`, and `psql`; these tools do not understand the
|
||||||
|
`postgresql+psycopg://` driver marker.
|
||||||
|
|
||||||
|
Bootstrap or upgrade the schema explicitly during deployment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export APP_ENV=prod
|
||||||
|
export ENABLED_MODULES=tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url "$DATABASE_URL"
|
||||||
|
```
|
||||||
|
|
||||||
|
Backup and restore-check before migration-bearing package changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pg_dump --format=custom \
|
||||||
|
--file "$PWD/runtime/govoplan-$(date +%Y%m%d%H%M%S).dump" \
|
||||||
|
"$GOVOPLAN_DATABASE_URL_PGTOOLS"
|
||||||
|
pg_restore --list "$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump" >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Restore a checked backup to the target database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pg_restore --clean --if-exists \
|
||||||
|
--dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" \
|
||||||
|
"$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump"
|
||||||
|
```
|
||||||
|
|
||||||
|
For local development, create the host database described in
|
||||||
|
`dev/postgres/README.md`, then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python -m govoplan_core.commands.init_db \
|
||||||
|
--database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \
|
||||||
|
--with-dev-data
|
||||||
|
./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload
|
||||||
|
scripts/launch-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
For disposable local validation against a throwaway PostgreSQL instance, use
|
||||||
|
the bundled PostgreSQL testbed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
The integration check runs migrations and startup smoke checks across the
|
||||||
|
standard module permutations. `--reset-schema` is destructive and belongs only
|
||||||
|
on throwaway databases.
|
||||||
|
|
||||||
|
### Broker And Workers
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `REDIS_URL` | `redis://redis:6379/0` | Celery broker/result backend when async workers are enabled. |
|
||||||
|
| `CELERY_ENABLED` | `false` | Local/dev can send synchronously. Production campaign delivery should run workers and set this to `true`. |
|
||||||
|
| `CELERY_QUEUES` | `send_email,append_sent,default` | Queue list expected by worker/process manager definitions. |
|
||||||
|
|
||||||
|
Worker command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m celery -A govoplan_core.celery_app:celery worker \
|
||||||
|
--queues send_email,append_sent,default \
|
||||||
|
--loglevel INFO
|
||||||
|
```
|
||||||
|
|
||||||
|
### Storage
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `FILE_STORAGE_BACKEND` | `local` | Use `local` for dev/small deployments; use object storage when files must scale independently. |
|
||||||
|
| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Must live on durable storage and be backed up when `FILE_STORAGE_BACKEND=local`. |
|
||||||
|
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Read-only fallback roots for migrated local files. |
|
||||||
|
| `FILE_STORAGE_S3_ENDPOINT_URL` | empty | Object-store endpoint for the files module. |
|
||||||
|
| `FILE_STORAGE_S3_REGION` | empty | Object-store region. |
|
||||||
|
| `FILE_STORAGE_S3_ACCESS_KEY_ID` | empty | Secret; inject through deployment environment. |
|
||||||
|
| `FILE_STORAGE_S3_SECRET_ACCESS_KEY` | empty | Secret; inject through deployment environment. |
|
||||||
|
| `FILE_STORAGE_S3_BUCKET` | `files` | Managed-file object bucket. |
|
||||||
|
|
||||||
|
Legacy `S3_*` settings remain for older storage paths but new deployments should
|
||||||
|
prefer `FILE_STORAGE_*`.
|
||||||
|
|
||||||
|
### HTTP, Cookies, And Base URLs
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `CORS_ORIGINS` | local dev origins | Set to the exact WebUI origins in staging/production. |
|
||||||
|
| `AUTH_SESSION_COOKIE_NAME` | configured default | Change only through a controlled rollout because it logs users out. |
|
||||||
|
| `AUTH_CSRF_COOKIE_NAME` | configured default | Must match WebUI/API deployment. |
|
||||||
|
| `AUTH_COOKIE_SECURE` | `false` | Set `true` behind HTTPS. |
|
||||||
|
| `AUTH_COOKIE_SAMESITE` | `lax` | Use a stricter value only after testing login and CSRF flows. |
|
||||||
|
| `AUTH_COOKIE_DOMAIN` | empty | Set only when the API and WebUI intentionally share a parent domain. |
|
||||||
|
|
||||||
|
Public URLs are currently supplied by deployment/reverse-proxy configuration and
|
||||||
|
module settings. Do not hardcode them in core; configuration packages should ask
|
||||||
|
for portal, WebUI, postbox, and notification URLs when they become relevant.
|
||||||
|
|
||||||
|
### Module Catalogs, Licenses, And Trust Roots
|
||||||
|
|
||||||
|
| Setting | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` or `GOVOPLAN_MODULE_PACKAGE_CATALOG` | Module package catalog source. |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE` | Preferred production keyring path. |
|
||||||
|
| `GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL` | Approved catalog channel, for example `stable`. |
|
||||||
|
| `GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE` | Trusted license issuer keyring path. |
|
||||||
|
| `GOVOPLAN_LICENSE_ENFORCEMENT` | Enables license enforcement when set to `true`. |
|
||||||
|
|
||||||
|
Trust roots are deployment-managed and should not be editable through the
|
||||||
|
running WebUI.
|
||||||
|
|
||||||
|
### Mail Test Credentials
|
||||||
|
|
||||||
|
Dedicated SMTP/IMAP test credentials belong to the mail/campaign test-bed
|
||||||
|
configuration, not the core runtime contract. Store them in a local ignored
|
||||||
|
`.env` file for the test bed or in CI secrets. Required values are:
|
||||||
|
|
||||||
|
- SMTP host, port, TLS mode, username, password, and envelope/from address.
|
||||||
|
- IMAP host, port, TLS mode, username, password, and append folder.
|
||||||
|
- At least one recipient mailbox that is safe for automated send tests.
|
||||||
|
|
||||||
|
## First Deployment Flow
|
||||||
|
|
||||||
|
1. Create an environment file or secret set with the runtime contract above.
|
||||||
|
2. Install the tagged core and module packages from `requirements-release.txt`.
|
||||||
|
3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt
|
||||||
|
artifact from the same release tag.
|
||||||
|
4. Run database migrations with the target `DATABASE_URL`.
|
||||||
|
5. Create the first tenant and system owner through the controlled bootstrap or
|
||||||
|
one-time admin command for the deployment.
|
||||||
|
6. Start the API service with `govoplan_core.server.app:app`.
|
||||||
|
7. Start workers when `CELERY_ENABLED=true`.
|
||||||
|
8. Start the WebUI/reverse proxy and verify CORS/cookie settings.
|
||||||
|
9. Open Admin > System > Modules, verify enabled modules, and save desired
|
||||||
|
module state if it differs from `ENABLED_MODULES`.
|
||||||
|
10. Run health checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS http://127.0.0.1:8000/health
|
||||||
|
curl -fsS http://127.0.0.1:8000/api/v1/platform/modules
|
||||||
|
```
|
||||||
|
|
||||||
|
Authenticated health details require `system:settings:read`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsS -H "X-API-Key: $GOVOPLAN_HEALTH_API_KEY" \
|
||||||
|
http://127.0.0.1:8000/health/details
|
||||||
|
```
|
||||||
|
|
||||||
|
## Production-Like Dev Profile
|
||||||
|
|
||||||
|
Use this profile to verify deployment behavior without publishing packages or
|
||||||
|
using real production credentials. The canonical launcher keeps API, worker, and
|
||||||
|
WebUI code in the editable repositories while Docker provides PostgreSQL and
|
||||||
|
Redis:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The launcher uses `dev/production-like/.env` when present, otherwise the checked
|
||||||
|
in `.env.example`. It runs:
|
||||||
|
|
||||||
|
- PostgreSQL on `127.0.0.1:55433`
|
||||||
|
- Redis on `127.0.0.1:56379`
|
||||||
|
- explicit `ENABLED_MODULES`
|
||||||
|
- explicit migrations and `--with-dev-data` bootstrap
|
||||||
|
- API via the module-aware devserver
|
||||||
|
- a Celery worker for `send_email,append_sent,default`
|
||||||
|
- WebUI through the Vite dev server
|
||||||
|
- durable local files under `runtime/production-like/files`
|
||||||
|
|
||||||
|
This profile validates explicit migration execution, config loading, module
|
||||||
|
discovery, route aggregation, local storage paths, Redis broker connectivity,
|
||||||
|
worker heartbeats, and health/readiness startup without real production
|
||||||
|
credentials. It does not replace a managed PostgreSQL/Redis/WebUI/worker
|
||||||
|
deployment test.
|
||||||
|
|
||||||
|
To stop PostgreSQL and Redis when the launcher exits:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Module Install/Uninstall Operations
|
||||||
|
|
||||||
|
Use Admin > System > Modules for planning. The running API server validates and
|
||||||
|
queues install plans; it does not run pip/npm or restart itself from an HTTP
|
||||||
|
request. Package mutation belongs to the trusted installer CLI/daemon in an
|
||||||
|
operator shell while maintenance mode is active.
|
||||||
|
|
||||||
|
Preflight from the server shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --format shell
|
||||||
|
```
|
||||||
|
|
||||||
|
Apply a prepared plan directly from a controlled shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --apply --build-webui
|
||||||
|
```
|
||||||
|
|
||||||
|
For production-like runs, prefer supervised mode with migrations, database
|
||||||
|
backup/restore hooks, restart commands, and health checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--supervise \
|
||||||
|
--migrate \
|
||||||
|
--restart-command 'systemctl restart govoplan-api' \
|
||||||
|
--restart-command 'systemctl restart govoplan-worker' \
|
||||||
|
--health-url http://127.0.0.1:8000/health \
|
||||||
|
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
||||||
|
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"'
|
||||||
|
```
|
||||||
|
|
||||||
|
To let the admin UI submit work without executing package managers inside the
|
||||||
|
API process, run the daemon in a separate operator shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--daemon \
|
||||||
|
--migrate \
|
||||||
|
--build-webui \
|
||||||
|
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
||||||
|
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
||||||
|
--health-url http://127.0.0.1:8000/health \
|
||||||
|
--restart-command '<restart govoplan server>'
|
||||||
|
```
|
||||||
|
|
||||||
|
The daemon claims one queued request at a time and writes request/run records
|
||||||
|
below `runtime/module-installer`. For process-manager one-shot usage or tests,
|
||||||
|
use `--daemon-once`. Check daemon status with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --daemon-status --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer uses a runtime lock, snapshots `pip freeze` plus WebUI package
|
||||||
|
files, writes a run record, and marks planned rows as applied only after all
|
||||||
|
commands succeed. With `--migrate`, SQLite databases are backed up through
|
||||||
|
SQLite's backup API; non-SQLite databases require
|
||||||
|
`--database-backup-command`, `--database-restore-check-command`, and
|
||||||
|
`--database-restore-command`.
|
||||||
|
|
||||||
|
Database hook commands receive:
|
||||||
|
|
||||||
|
- `GOVOPLAN_INSTALLER_RUN_DIR`
|
||||||
|
- `GOVOPLAN_DATABASE_URL`
|
||||||
|
- `GOVOPLAN_DATABASE_URL_PGTOOLS` for PostgreSQL tools
|
||||||
|
- `GOVOPLAN_DATABASE_BACKUP_PATH`
|
||||||
|
- `GOVOPLAN_DATABASE_BACKUP_METADATA`
|
||||||
|
|
||||||
|
Avoid embedding secrets directly in commands; prefer environment variables,
|
||||||
|
service credentials, or deployment-local secret injection.
|
||||||
|
|
||||||
|
Inspect installer history and lock state from the operator shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --list-runs --format json
|
||||||
|
govoplan-module-installer --show-run <run-id> --format json
|
||||||
|
govoplan-module-installer --lock-status --format json
|
||||||
|
govoplan-module-installer --list-requests --format json
|
||||||
|
govoplan-module-installer --show-request <request-id> --format json
|
||||||
|
govoplan-module-installer --cancel-request <request-id> --format json
|
||||||
|
govoplan-module-installer --retry-request <request-id> --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Rollback uses the saved run snapshot:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer --rollback <run-id>
|
||||||
|
govoplan-module-installer --rollback <run-id> --database-restore-command '<override restore command>'
|
||||||
|
```
|
||||||
|
|
||||||
|
Uninstall is non-destructive by default. A planned uninstall row can set
|
||||||
|
`destroy_data: true` to request destructive module retirement. The module must
|
||||||
|
provide an automated retirement provider, and the installer snapshots the
|
||||||
|
database before dropping module-owned tables.
|
||||||
|
|
||||||
|
Run the rollback drill before relying on installer automation in a new
|
||||||
|
environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/module-installer-rollback-drill.py --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The drill uses temporary SQLite databases and simulated package commands. It
|
||||||
|
does not install or uninstall real packages. It exercises:
|
||||||
|
|
||||||
|
- package command failure followed by supervised rollback;
|
||||||
|
- migration failure with a SQLite database snapshot;
|
||||||
|
- restart-command failure;
|
||||||
|
- health timeout after restart;
|
||||||
|
- destructive retirement executor failure with database rollback;
|
||||||
|
- PostgreSQL-style backup, restore-check, and restore hooks;
|
||||||
|
- daemon heartbeat, request queue claim/update, retry/cancel, and stale lock
|
||||||
|
detection/removal.
|
||||||
|
|
||||||
|
See `RELEASE_DEPENDENCIES.md` for release package refs, migration baseline
|
||||||
|
checks, catalog trust, signing, keyring, replay, and license operation.
|
||||||
|
|
||||||
|
## Operator Checklist
|
||||||
|
|
||||||
|
- Runtime secrets are injected outside git.
|
||||||
|
- `MASTER_KEY_B64` is set and backed up securely.
|
||||||
|
- Database backup and restore commands are tested.
|
||||||
|
- File/object storage is durable and backed up.
|
||||||
|
- `CORS_ORIGINS` and cookie settings match the deployed WebUI origin.
|
||||||
|
- Redis and workers are running before `CELERY_ENABLED=true`.
|
||||||
|
- Module catalog and license keyrings are pinned locally.
|
||||||
|
- Health endpoints are monitored.
|
||||||
|
- Test SMTP/IMAP credentials are non-production and isolated.
|
||||||
|
- Module installer rollback drill has passed in the deployment environment.
|
||||||
51
docs/DOCUMENTATION_MAP.md
Normal file
51
docs/DOCUMENTATION_MAP.md
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
# GovOPlaN Documentation Map
|
||||||
|
|
||||||
|
This map defines the source-of-truth documents for the current repository docs.
|
||||||
|
Use it to avoid duplicating long procedures across architecture, release,
|
||||||
|
operator, and roadmap pages.
|
||||||
|
|
||||||
|
## Core Platform
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Module architecture and kernel contracts | `MODULE_ARCHITECTURE.md` | Stable module contracts, API efficiency contracts, durable boundary decisions, lifecycle, and WebUI contribution rules. |
|
||||||
|
| RBAC and resource access | `ACCESS_RBAC_MODEL.md` | Current permission, role, API-key, and resource-access model. |
|
||||||
|
| Governance hierarchy | `GOVERNANCE_MODEL.md` | System, tenant, user/group, campaign policy inheritance and admin UI structure. |
|
||||||
|
| Policy decision DTOs and provenance | `POLICY_CONTRACTS.md` | Shared explain/provenance shape; module-specific policy docs should link here. |
|
||||||
|
| Events and audit trace context | `EVENTS_AND_AUDIT.md` | Event dispatch semantics and audit payload conventions. |
|
||||||
|
| Action/effect automation layer | `ACTION_EFFECT_AUTOMATION_LAYER.md` | Action/effect contracts, consequence preview, runner semantics, and module boundary for automation. |
|
||||||
|
| Postbox E2EE target architecture | `POSTBOX_E2EE_ARCHITECTURE.md` | Strategic encrypted postbox/mailbox model, key ownership, role mailbox semantics, and retraction limits. |
|
||||||
|
|
||||||
|
## Release And Operations
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Runtime configuration and operator flow | `DEPLOYMENT_OPERATOR_GUIDE.md` | Production/staging configuration, migrations, backups, installer operation, and rollback drill. |
|
||||||
|
| Release dependencies and catalogs | `RELEASE_DEPENDENCIES.md` | Release package refs, migration baselines, release lockfiles, catalog trust/licensing, catalog publishing, and release checklist. |
|
||||||
|
| Dependency vulnerability audits | `DEPENDENCY_AUDITS.md` | Local and CI audit commands plus dated audit result notes. |
|
||||||
|
| Remote WebUI bundle design | `REMOTE_WEBUI_BUNDLES.md` | Experimental controlled-deployment design; normal releases still use package builds. |
|
||||||
|
|
||||||
|
## Product And Module Planning
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Product roadmap and module routing | `GOVOPLAN_MASTER_ROADMAP.md` | Product-level sequencing, implementation gates, issue routing, and missing-module decisions. |
|
||||||
|
| UI/UX decisions | `UI_UX_DECISION_LEDGER.md` | Binding guided-UI decisions, open decisions, impact index, and review checklist. |
|
||||||
|
| Interface ethics and design doctrine | `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` | Product-level doctrine for context, decision, consequence, contestability, responsibility, and traceability. |
|
||||||
|
| Public-sector integration posture | `PUBLIC_SECTOR_INTEGRATION_STRATEGY.md` | Strategy index; executable target inventory lives in `govoplan-connectors`. |
|
||||||
|
| Configuration packages | `CONFIGURATION_PACKAGES.md` | Package model, provider contract, import/export flow, and tracking slices. |
|
||||||
|
|
||||||
|
## Workflow Docs
|
||||||
|
|
||||||
|
| Topic | Canonical document | Notes |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Gitea issues and wiki sync | `GITEA_ISSUES.md` | Issue labels, imports, wiki mirroring, and Codex state updates. |
|
||||||
|
| Codex local workflow | `CODEX_WORKFLOW.md` | Local agent setup and focused verification commands. |
|
||||||
|
|
||||||
|
## Cross-Repo Rule
|
||||||
|
|
||||||
|
Core docs may keep strategy, kernel contracts, and routing decisions. Module
|
||||||
|
repositories should own executable module behavior, concrete API/UI contracts,
|
||||||
|
and operator notes for their own module. When content spans both, keep the
|
||||||
|
durable decision in core and link to the module document for implementation
|
||||||
|
details.
|
||||||
180
docs/EVENTS_AND_AUDIT.md
Normal file
180
docs/EVENTS_AND_AUDIT.md
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
# Events And Audit
|
||||||
|
|
||||||
|
GovOPlaN uses a small kernel event contract first, not a broad command bus.
|
||||||
|
Commands remain module-owned application-service methods or API endpoints until
|
||||||
|
there is a concrete need for durable asynchronous command orchestration. Events
|
||||||
|
are facts about completed work and are safe for audit, projections, optional
|
||||||
|
module reactions, and operator diagnostics.
|
||||||
|
|
||||||
|
## Production Transport Decision
|
||||||
|
|
||||||
|
The first production target is a **database outbox plus in-process immediate
|
||||||
|
dispatch**:
|
||||||
|
|
||||||
|
- Use `govoplan_core.core.events.PlatformEvent` for domain and platform events.
|
||||||
|
- Use `EventBus` as the in-process dispatch contract for same-process module
|
||||||
|
reactions that are safe to run inline.
|
||||||
|
- Persist durable integration/workflow events through a database outbox before
|
||||||
|
acknowledging the state change that produced them.
|
||||||
|
- Drain the outbox through a small dispatcher process. The dispatcher may call
|
||||||
|
in-process handlers in the same deployment first, but its storage contract is
|
||||||
|
database-backed.
|
||||||
|
- Treat Redis/Celery as worker/job infrastructure, not as the authoritative
|
||||||
|
first event transport. A Celery dispatcher can consume the outbox later.
|
||||||
|
- Keep the dispatch implementation pluggable behind the `PlatformEvent`
|
||||||
|
envelope so a future message broker can be added without changing event
|
||||||
|
producers.
|
||||||
|
- Keep commands out of the kernel until workflows need retryable, durable,
|
||||||
|
operator-visible command records.
|
||||||
|
|
||||||
|
This keeps the first contract small, PostgreSQL-friendly, auditable, and
|
||||||
|
recoverable after process crashes. It also avoids making Redis a correctness
|
||||||
|
dependency for deployments that only need synchronous mail/tests or light
|
||||||
|
background work.
|
||||||
|
|
||||||
|
## Dispatch Semantics
|
||||||
|
|
||||||
|
Event producers should write their domain state and outbox event in the same
|
||||||
|
database transaction wherever possible. Handlers must be idempotent because the
|
||||||
|
outbox dispatcher can retry after a crash or timeout.
|
||||||
|
|
||||||
|
Recommended first outbox columns:
|
||||||
|
|
||||||
|
- `event_id`, `event_type`, `module_id`
|
||||||
|
- `correlation_id`, `causation_id`
|
||||||
|
- `payload`, `occurred_at`
|
||||||
|
- `available_at`, `attempt_count`, `claimed_at`, `claim_token`
|
||||||
|
- `processed_at`, `last_error`
|
||||||
|
|
||||||
|
Inline `EventBus` handlers are allowed only for non-critical local reactions.
|
||||||
|
Anything that must survive process failure, restart, package update, or worker
|
||||||
|
redeployment belongs in the outbox.
|
||||||
|
|
||||||
|
## Trace IDs
|
||||||
|
|
||||||
|
Every `PlatformEvent` has:
|
||||||
|
|
||||||
|
- `event_id`: unique ID for that event.
|
||||||
|
- `correlation_id`: stable ID for the whole request, workflow, or job.
|
||||||
|
- `causation_id`: the event ID or external operation ID that caused this
|
||||||
|
event.
|
||||||
|
|
||||||
|
The FastAPI app factory creates an event context for every request. It accepts
|
||||||
|
`X-Correlation-ID` or `X-Request-ID` when the value is a compact safe trace ID,
|
||||||
|
otherwise it generates a new ID. Responses include `X-Correlation-ID`.
|
||||||
|
|
||||||
|
Audit logging reads the current event context and stores trace IDs in
|
||||||
|
`details._trace`. Callers can also pass explicit `correlation_id` and
|
||||||
|
`causation_id` to `audit_event` or `audit_from_principal`.
|
||||||
|
|
||||||
|
Admin and lifecycle code should use the compact operational detail shape
|
||||||
|
documented in `govoplan-audit/docs/AUDIT_TRACE_CONTEXT.md`. The core
|
||||||
|
`audit_operation_context` helper preserves `module_id`, `request_id`, `run_id`,
|
||||||
|
`outcome`, and `_trace` while applying the shared audit redaction pass to
|
||||||
|
additional detail values.
|
||||||
|
|
||||||
|
## Audit MVP Boundary
|
||||||
|
|
||||||
|
`govoplan-audit` owns:
|
||||||
|
|
||||||
|
- the `audit_log` table and audit API routes
|
||||||
|
- audit route contribution through its module manifest
|
||||||
|
- audit retention behavior in cooperation with policy/retention settings
|
||||||
|
- future audit sink/export capability implementations
|
||||||
|
|
||||||
|
`govoplan-core` owns:
|
||||||
|
|
||||||
|
- `AuditEvent` and `AuditSink` protocol contracts
|
||||||
|
- request and event trace context
|
||||||
|
- the compatibility `audit_event` helper while routes are still migrating
|
||||||
|
- retention orchestration that calls module capabilities
|
||||||
|
|
||||||
|
Feature modules should record audit facts through a small audit API or future
|
||||||
|
`audit.sink` capability. They should not import audit storage internals.
|
||||||
|
|
||||||
|
## Initial Domain Event Inventory
|
||||||
|
|
||||||
|
Access:
|
||||||
|
|
||||||
|
- `access.account.created`
|
||||||
|
- `access.account.updated`
|
||||||
|
- `access.membership.created`
|
||||||
|
- `access.membership.updated`
|
||||||
|
- `access.group.created`
|
||||||
|
- `access.group.updated`
|
||||||
|
- `access.role.created`
|
||||||
|
- `access.role.updated`
|
||||||
|
- `access.role.deleted`
|
||||||
|
- `access.api_key.created`
|
||||||
|
- `access.api_key.revoked`
|
||||||
|
- `access.session.created`
|
||||||
|
- `access.session.revoked`
|
||||||
|
|
||||||
|
Tenancy:
|
||||||
|
|
||||||
|
- `tenancy.tenant.created`
|
||||||
|
- `tenancy.tenant.updated`
|
||||||
|
- `tenancy.tenant.suspended`
|
||||||
|
- `tenancy.tenant.reactivated`
|
||||||
|
- `tenancy.tenant.delete_requested`
|
||||||
|
- `tenancy.tenant.delete_blocked`
|
||||||
|
- `tenancy.tenant.deleted`
|
||||||
|
|
||||||
|
Policy:
|
||||||
|
|
||||||
|
- `policy.system.updated`
|
||||||
|
- `policy.tenant.updated`
|
||||||
|
- `policy.user.updated`
|
||||||
|
- `policy.group.updated`
|
||||||
|
- `policy.campaign.updated`
|
||||||
|
- `policy.effective_policy.changed`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
|
||||||
|
- `files.file.uploaded`
|
||||||
|
- `files.file.renamed`
|
||||||
|
- `files.file.deleted`
|
||||||
|
- `files.file.frozen`
|
||||||
|
- `files.share.created`
|
||||||
|
- `files.share.revoked`
|
||||||
|
- `files.connector.imported`
|
||||||
|
- `files.connector.access_denied`
|
||||||
|
|
||||||
|
Mail:
|
||||||
|
|
||||||
|
- `mail.profile.created`
|
||||||
|
- `mail.profile.updated`
|
||||||
|
- `mail.profile.credentials_rotated`
|
||||||
|
- `mail.profile.tested`
|
||||||
|
- `mail.message.sent`
|
||||||
|
- `mail.message.send_failed`
|
||||||
|
- `mail.imap.appended`
|
||||||
|
- `mail.imap.append_failed`
|
||||||
|
- `mail.mailbox.message_seen`
|
||||||
|
|
||||||
|
Campaign:
|
||||||
|
|
||||||
|
- `campaign.created`
|
||||||
|
- `campaign.version.created`
|
||||||
|
- `campaign.validated`
|
||||||
|
- `campaign.built`
|
||||||
|
- `campaign.reviewed`
|
||||||
|
- `campaign.queued`
|
||||||
|
- `campaign.send_started`
|
||||||
|
- `campaign.recipient_attempted`
|
||||||
|
- `campaign.recipient_delivered`
|
||||||
|
- `campaign.recipient_failed`
|
||||||
|
- `campaign.paused`
|
||||||
|
- `campaign.resumed`
|
||||||
|
- `campaign.cancelled`
|
||||||
|
- `campaign.report.exported`
|
||||||
|
|
||||||
|
## Event Payload Rules
|
||||||
|
|
||||||
|
- Payloads must be JSON-serializable.
|
||||||
|
- Use stable IDs, not ORM objects.
|
||||||
|
- Include tenant ID when tenant-scoped.
|
||||||
|
- Include actor/principal references only as DTOs or primitive IDs.
|
||||||
|
- Do not include secrets, raw message bodies, full recipient lists, or file
|
||||||
|
content.
|
||||||
|
- Put large evidence in owning module storage and reference it by ID.
|
||||||
@@ -33,7 +33,15 @@ For a shared credentials file outside the target repository, pass `--env-file`:
|
|||||||
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply
|
||||||
```
|
```
|
||||||
|
|
||||||
Create a Gitea token with issue read/write access and label-management permission for the repository. On scoped-token instances, this usually means issue read/write and, if label writes are rejected, repository write permission too.
|
Create a Gitea token with issue read/write access and label-management
|
||||||
|
permission for the repository. On scoped-token instances, this usually means
|
||||||
|
issue read/write and, if label writes are rejected, repository write permission
|
||||||
|
too.
|
||||||
|
|
||||||
|
For GovOPlaN repositories, prefer organization labels for the shared taxonomy.
|
||||||
|
Creating or updating organization labels requires a token with
|
||||||
|
`write:organization`. Repository label management only needs repository label
|
||||||
|
permission, but it duplicates the taxonomy into each repository.
|
||||||
|
|
||||||
Preview and apply labels:
|
Preview and apply labels:
|
||||||
|
|
||||||
@@ -43,6 +51,19 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
./scripts/gitea-sync-labels.py --apply
|
./scripts/gitea-sync-labels.py --apply
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Preview and apply the shared taxonomy as organization labels:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env
|
||||||
|
./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The import helpers resolve repository labels and organization labels. Repository
|
||||||
|
labels win when a repository defines the same name locally, but a repository does
|
||||||
|
not need a local copy of every shared `type/*`, `status/*`, `priority/*`,
|
||||||
|
`module/*`, `area/*`, `source/*`, or `codex/*` label.
|
||||||
|
|
||||||
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`.
|
||||||
|
|
||||||
## Multiple Repositories And Workspaces
|
## Multiple Repositories And Workspaces
|
||||||
@@ -167,7 +188,10 @@ For a broader project import across all local repositories hosted on `git.add-id
|
|||||||
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
```
|
```
|
||||||
|
|
||||||
It scans repository and product-directory files with backlog-like names, creates the shared generic labels where needed, imports missing open work, and deduplicates reruns by hidden fingerprint and normalized title.
|
It scans repository and product-directory files with backlog-like names, resolves
|
||||||
|
shared labels from the organization label catalogue where available, creates only
|
||||||
|
missing fallback repository labels when needed, imports missing open work, and
|
||||||
|
deduplicates reruns by hidden fingerprint and normalized title.
|
||||||
|
|
||||||
## Mirroring Project Docs Into Gitea Wikis
|
## Mirroring Project Docs Into Gitea Wikis
|
||||||
|
|
||||||
@@ -184,6 +208,13 @@ Apply the wiki mirror:
|
|||||||
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply
|
||||||
```
|
```
|
||||||
|
|
||||||
|
After renaming or deleting repository docs, prune previously managed wiki pages
|
||||||
|
that no longer have a source file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply
|
||||||
|
```
|
||||||
|
|
||||||
The default apply path uses the Gitea wiki git repository, not one REST API
|
The default apply path uses the Gitea wiki git repository, not one REST API
|
||||||
request per page. It keeps a local checkout cache below
|
request per page. It keeps a local checkout cache below
|
||||||
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# Multi Seal Mail - Current System and Tenant Governance Model
|
# GovOPlaN Governance Model
|
||||||
|
|
||||||
**Updated:** 2026-06-16
|
**Updated:** 2026-07-09
|
||||||
**Current migration head:** `f5a6b7c8d9e0`
|
|
||||||
|
|
||||||
## Governance Rule
|
## Governance Rule
|
||||||
|
|
||||||
System policy is authoritative for tenants and all lower levels. Each lower level may only narrow what it inherits:
|
System policy is authoritative for tenants and all lower levels. Each lower
|
||||||
|
level may only narrow what it inherits:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
system
|
system
|
||||||
@@ -14,52 +14,65 @@ system
|
|||||||
-> campaign
|
-> campaign
|
||||||
```
|
```
|
||||||
|
|
||||||
Lower levels do not widen privileges, allowed profiles, retention durations or credential rights granted by a higher level.
|
Lower levels do not widen privileges, allowed profiles, retention durations, or
|
||||||
|
credential rights granted by a higher level.
|
||||||
|
|
||||||
## Administration Structure
|
## Administration Structure
|
||||||
|
|
||||||
|
GovOPlaN separates system administration from scoped configuration:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
SYSTEM
|
ADMINISTRATION
|
||||||
- Settings
|
- Modules
|
||||||
- Retention
|
- Packages
|
||||||
- Mail servers
|
- Maintenance
|
||||||
|
- Changes
|
||||||
|
|
||||||
|
GLOBAL
|
||||||
- Tenants
|
- Tenants
|
||||||
- Users
|
- Roles
|
||||||
- Groups
|
- Groups and users
|
||||||
- System roles
|
- File connectors
|
||||||
- Tenant roles
|
- Mail servers
|
||||||
- Audit
|
- API keys
|
||||||
|
- Retention
|
||||||
|
|
||||||
TENANT
|
TENANT
|
||||||
- Settings boundary
|
|
||||||
- Users
|
|
||||||
- Groups
|
|
||||||
- Roles
|
- Roles
|
||||||
- API keys
|
- Groups and users
|
||||||
|
- File connectors
|
||||||
- Mail servers
|
- Mail servers
|
||||||
|
- API keys
|
||||||
- Retention
|
- Retention
|
||||||
- Audit
|
|
||||||
|
|
||||||
USER
|
|
||||||
- User mail
|
|
||||||
- User retention
|
|
||||||
|
|
||||||
GROUP
|
GROUP
|
||||||
- Group mail
|
- File connectors
|
||||||
- Group retention
|
- Mail servers
|
||||||
|
- API keys
|
||||||
|
- Retention
|
||||||
|
|
||||||
|
USER
|
||||||
|
- File connectors
|
||||||
|
- Mail servers
|
||||||
|
- API keys
|
||||||
|
- Retention
|
||||||
```
|
```
|
||||||
|
|
||||||
There is no separate System access page. Compatibility access scopes remain in the backend for assignment/read boundaries.
|
System access scopes remain in the backend for assignment/read boundaries, but
|
||||||
|
the UI should present the configuration hierarchy rather than a separate
|
||||||
|
"system access" concept.
|
||||||
|
|
||||||
## Tenant Governance
|
## Tenant Governance
|
||||||
|
|
||||||
System settings define tenant defaults and whether tenants may narrow selected options. Tenant overrides can only restrict:
|
System settings define tenant defaults and whether tenants may narrow selected
|
||||||
|
options. Tenant overrides can only restrict:
|
||||||
|
|
||||||
- custom groups;
|
- custom groups;
|
||||||
- custom roles;
|
- custom roles;
|
||||||
- tenant API keys.
|
- tenant API keys.
|
||||||
|
|
||||||
The backend enforces that tenant governance cannot widen system-denied privileges.
|
The backend enforces that tenant governance cannot widen system-denied
|
||||||
|
privileges.
|
||||||
|
|
||||||
## Mail-Profile Governance
|
## Mail-Profile Governance
|
||||||
|
|
||||||
@@ -73,7 +86,10 @@ group
|
|||||||
campaign
|
campaign
|
||||||
```
|
```
|
||||||
|
|
||||||
Effective campaign profile availability follows campaign ownership. A campaign owned by a user resolves through system, tenant, that user and campaign policy. A group-owned campaign resolves through system, tenant, that group and campaign policy.
|
Effective campaign profile availability follows campaign ownership. A campaign
|
||||||
|
owned by a user resolves through system, tenant, that user, and campaign
|
||||||
|
policy. A group-owned campaign resolves through system, tenant, that group, and
|
||||||
|
campaign policy.
|
||||||
|
|
||||||
Policy semantics:
|
Policy semantics:
|
||||||
|
|
||||||
@@ -81,12 +97,18 @@ Policy semantics:
|
|||||||
- lower levels can further restrict the set;
|
- lower levels can further restrict the set;
|
||||||
- forced profiles mean the lower level must choose from the forced set;
|
- forced profiles mean the lower level must choose from the forced set;
|
||||||
- a forced set with one profile effectively enforces that profile;
|
- a forced set with one profile effectively enforces that profile;
|
||||||
- campaign-level profile creation is allowed only if the effective policy permits it;
|
- campaign-level profile creation is allowed only if the effective policy
|
||||||
- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels must inherit profile credentials, may inherit profile credentials, or must provide local credentials;
|
permits it;
|
||||||
- the lower-level override switch for `smtp_credentials.inherit` and `imap_credentials.inherit` controls whether descendants may change that inheritance decision;
|
- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels
|
||||||
|
must inherit profile credentials, may inherit profile credentials, or must
|
||||||
|
provide local credentials;
|
||||||
|
- the lower-level override switch for `smtp_credentials.inherit` and
|
||||||
|
`imap_credentials.inherit` controls whether descendants may change that
|
||||||
|
inheritance decision;
|
||||||
- deny patterns always win over allow patterns;
|
- deny patterns always win over allow patterns;
|
||||||
- empty or `*` allowlist means allow all except denied;
|
- empty or `*` allowlist means allow all except denied;
|
||||||
- non-empty allowlist means at least one allow rule must match and no deny rule may match.
|
- non-empty allowlist means at least one allow rule must match and no deny rule
|
||||||
|
may match.
|
||||||
|
|
||||||
Pattern targets:
|
Pattern targets:
|
||||||
|
|
||||||
@@ -98,7 +120,23 @@ From header
|
|||||||
recipient domains
|
recipient domains
|
||||||
```
|
```
|
||||||
|
|
||||||
Ownership transfer is intentionally deferred as a two-step workflow: original owner initiates, new owner accepts and reselects/repairs the mail profile if their effective policy requires it.
|
Ownership transfer is intentionally deferred as a two-step workflow: original
|
||||||
|
owner initiates, new owner accepts and reselects/repairs the mail profile if
|
||||||
|
their effective policy requires it.
|
||||||
|
|
||||||
|
## File-Connector Governance
|
||||||
|
|
||||||
|
File connector profiles and credentials are separated. Profiles describe
|
||||||
|
external endpoints; credentials bind authentication material and policy to a
|
||||||
|
scope. Concrete linked folders appear as file spaces in the files module.
|
||||||
|
|
||||||
|
Governance follows the same inheritance shape as mail:
|
||||||
|
|
||||||
|
- system and tenant policy can permit, require, or forbid lower-level
|
||||||
|
connections/credentials;
|
||||||
|
- user or group ownership controls which spaces appear to principals;
|
||||||
|
- spaces inherit endpoint and credential policy from their connector profile;
|
||||||
|
- connector health and credential tests must not reveal plaintext secrets.
|
||||||
|
|
||||||
## Retention Governance
|
## Retention Governance
|
||||||
|
|
||||||
@@ -120,20 +158,27 @@ Rules:
|
|||||||
|
|
||||||
- system may set concrete defaults or unlimited retention;
|
- system may set concrete defaults or unlimited retention;
|
||||||
- system exposes allow-limiting toggles per field;
|
- system exposes allow-limiting toggles per field;
|
||||||
- tenants, users/groups and campaigns may only shorten inherited retention where the parent allows limiting;
|
- tenants, users/groups, and campaigns may only shorten inherited retention
|
||||||
|
where the parent allows limiting;
|
||||||
- blank lower-level values inherit;
|
- blank lower-level values inherit;
|
||||||
- mock mailbox retention is currently system-level because mock mailbox records do not yet carry tenant/campaign ownership metadata;
|
- mock mailbox retention is currently system-level because mock mailbox records
|
||||||
- dry-run/apply retention actions report affected classes before destructive cleanup.
|
do not yet carry tenant/campaign ownership metadata;
|
||||||
|
- dry-run/apply retention actions report affected classes before destructive
|
||||||
|
cleanup.
|
||||||
|
|
||||||
## Role Definitions and Assignments
|
## Role Definitions And Assignments
|
||||||
|
|
||||||
### System roles
|
### System Roles
|
||||||
|
|
||||||
System roles define instance-wide permissions. `system:*` is stored as one wildcard and displayed as granting the full system catalogue. System owner is protected.
|
System roles define instance-wide permissions. `system:*` is stored as one
|
||||||
|
wildcard and displayed as granting the full system catalogue. System owner is
|
||||||
|
protected.
|
||||||
|
|
||||||
### Tenant roles
|
### Tenant Roles
|
||||||
|
|
||||||
Tenant roles can be system-governed templates or tenant-local definitions, subject to system tenant-governance settings and actor delegation ceilings. Wildcard counts are expanded against the canonical tenant catalogue.
|
Tenant roles can be system-governed templates or tenant-local definitions,
|
||||||
|
subject to system tenant-governance settings and actor delegation ceilings.
|
||||||
|
Wildcard counts are expanded against the canonical tenant catalogue.
|
||||||
|
|
||||||
## Audit Access
|
## Audit Access
|
||||||
|
|
||||||
@@ -144,15 +189,17 @@ system audit -> system:audit:read
|
|||||||
tenant audit -> active tenant + audit:read
|
tenant audit -> active tenant + audit:read
|
||||||
```
|
```
|
||||||
|
|
||||||
Audit pages use server pagination, filtering and bounded grids.
|
Audit pages use server pagination, filtering, and bounded grids.
|
||||||
|
|
||||||
## Tenant Switching
|
## Tenant Switching
|
||||||
|
|
||||||
Tenant switching preserves the current URL when possible and falls back when a route/resource is not accessible in the new tenant context.
|
Tenant switching preserves the current URL when possible and falls back when a
|
||||||
|
route/resource is not accessible in the new tenant context.
|
||||||
|
|
||||||
The tenant selector is hidden for ordinary single-tenant accounts and visible for multi-tenant or system tenant-management contexts.
|
The tenant selector is hidden for ordinary single-tenant accounts and visible
|
||||||
|
for multi-tenant or system tenant-management contexts.
|
||||||
|
|
||||||
## DataGrid Contract in Administration
|
## Administration DataGrid Contract
|
||||||
|
|
||||||
Admin lists use bounded container grids:
|
Admin lists use bounded container grids:
|
||||||
|
|
||||||
@@ -164,14 +211,12 @@ Admin lists use bounded container grids:
|
|||||||
- sticky headers where needed;
|
- sticky headers where needed;
|
||||||
- server pagination for audit.
|
- server pagination for audit.
|
||||||
|
|
||||||
## Still Deferred
|
## Deferred Work
|
||||||
|
|
||||||
- real SMTP/IMAP test-bed verification and operator runbook;
|
- real SMTP/IMAP test-bed verification and operator runbook;
|
||||||
- recipient import with column mapping;
|
- recipient import with column mapping;
|
||||||
- Seafile/external connector governance;
|
|
||||||
- system/tenant/group/user file-space hierarchy and external storage hierarchy;
|
|
||||||
- session/device revocation UI;
|
- session/device revocation UI;
|
||||||
- backup/restore, monitoring and update procedures;
|
- backup/restore, monitoring, and update procedures;
|
||||||
- DSAR workflows and evidence bundle verifier;
|
- DSAR workflows and evidence bundle verifier;
|
||||||
- campaign ownership transfer workflow;
|
- campaign ownership transfer workflow;
|
||||||
- policy impact analysis before delete/disable/unshare/change;
|
- policy impact analysis before delete/disable/unshare/change;
|
||||||
663
docs/GOVOPLAN_MASTER_ROADMAP.md
Normal file
663
docs/GOVOPLAN_MASTER_ROADMAP.md
Normal file
@@ -0,0 +1,663 @@
|
|||||||
|
# GovOPlaN Master Roadmap
|
||||||
|
|
||||||
|
This roadmap is the durable product north star and sequencing guide for
|
||||||
|
GovOPlaN as a modular platform for administrative operations. It keeps the
|
||||||
|
product moving without turning every possible public-sector need into an
|
||||||
|
immediate implementation track.
|
||||||
|
|
||||||
|
Use this document for product direction, sequencing, and module routing. Issues
|
||||||
|
are the active backlog; this document is durable planning context and should be
|
||||||
|
mirrored to the Gitea wiki.
|
||||||
|
|
||||||
|
## Product Thesis
|
||||||
|
|
||||||
|
GovOPlaN should become a configurable operations platform for public
|
||||||
|
institutions. It should help an institution model real administrative
|
||||||
|
procedures, connect existing systems, keep durable evidence, and explain the
|
||||||
|
configured system to users.
|
||||||
|
|
||||||
|
The goal is not to replace every existing system. GovOPlaN should connect
|
||||||
|
existing systems, provide better workflows where the current landscape is weak,
|
||||||
|
and make administrative processes configurable, auditable, and reusable.
|
||||||
|
|
||||||
|
The product should first provide a reliable administrative spine:
|
||||||
|
|
||||||
|
- identities, roles, tenants, policy, audit, and governance
|
||||||
|
- forms, files, cases, workflow, tasks, templates, and records
|
||||||
|
- postboxes, notifications, mail, portal, appointments, and booking
|
||||||
|
- identity trust, role-bound postboxes, and eventually encrypted administrative
|
||||||
|
communication
|
||||||
|
- configuration packages that assemble modules into repeatable procedures
|
||||||
|
- docs that explain the configured system, not the full theoretical product
|
||||||
|
|
||||||
|
Domain modules should come after the spine can run a reference procedure end to
|
||||||
|
end.
|
||||||
|
|
||||||
|
The platform should be a governance-capable runtime for modules, connectors,
|
||||||
|
configuration, and administrative decisions. The kernel must stay free of domain
|
||||||
|
semantics while still providing the contracts needed for modules to explain what
|
||||||
|
they do, what they require, which effects they create, and how operators can
|
||||||
|
verify or reverse those effects.
|
||||||
|
|
||||||
|
## Design Principles
|
||||||
|
|
||||||
|
- Modules must stay independently installable, enableable, and disableable.
|
||||||
|
- Cross-module behavior should use core-mediated capabilities, commands,
|
||||||
|
events, DTOs, and UI contribution points rather than direct imports.
|
||||||
|
- Configuration packages should turn installed modules into concrete, reusable
|
||||||
|
administrative processes.
|
||||||
|
- Operators should be able to configure the platform through the UI.
|
||||||
|
- Every powerful configuration path needs preflight, preview, audit, rollback,
|
||||||
|
RBAC, and policy checks.
|
||||||
|
- Context, decision, consequence, and traceability must be visible together for
|
||||||
|
consequential actions. The full doctrine lives in
|
||||||
|
`INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md`.
|
||||||
|
- Automation must use governed action/effect contracts, not hidden side
|
||||||
|
effects. The first automation layer is defined in
|
||||||
|
`ACTION_EFFECT_AUTOMATION_LAYER.md` and should start in `govoplan-workflow`
|
||||||
|
unless a separate automation module becomes justified.
|
||||||
|
- Encrypted postboxes are a strategic target. Early postbox, access, and
|
||||||
|
identity-trust contracts should stay compatible with the E2EE architecture in
|
||||||
|
`POSTBOX_E2EE_ARCHITECTURE.md`.
|
||||||
|
- Integration should be a first-class product path: connect to existing
|
||||||
|
systems, consume their data, and publish governed outputs back to them.
|
||||||
|
- GovOPlaN should scale from a small local installation to a larger deployment
|
||||||
|
with separately scalable web, API, worker, storage, and database components.
|
||||||
|
|
||||||
|
## User Experience Direction
|
||||||
|
|
||||||
|
GovOPlaN should expose the full power of the platform without forcing
|
||||||
|
non-technical users to face every field, flag, and internal representation at
|
||||||
|
once. The default experience should feel guided, explainable, and calm. Expert
|
||||||
|
depth should remain available, but it should be layered behind deliberate
|
||||||
|
interaction patterns.
|
||||||
|
|
||||||
|
Core UX rules:
|
||||||
|
|
||||||
|
- Use progressive disclosure. Common decisions stay visible; advanced,
|
||||||
|
hazardous, or rarely used options live in collapsed panels, secondary steps,
|
||||||
|
or explicit advanced areas.
|
||||||
|
- Do not use raw JSON as the primary configuration UI. Every configurable value
|
||||||
|
should have an appropriate control, validation, and plain-language help.
|
||||||
|
Import/export and diagnostics may show JSON as a secondary artifact.
|
||||||
|
- Prefer guided flows over option dumps. Connector setup, package import,
|
||||||
|
module installation, policy changes, and destructive operations should use
|
||||||
|
wizards that explain what is happening, why it matters, and what will happen
|
||||||
|
next.
|
||||||
|
- Discover values when the system can infer them. For example, a Nextcloud file
|
||||||
|
connection should start with the base URL, discover the WebDAV endpoint, and
|
||||||
|
fill technical fields for review instead of asking the user to know them
|
||||||
|
upfront.
|
||||||
|
- Make explanations always available without making every screen verbose.
|
||||||
|
Inline helper text should be short; richer explanations should be reachable
|
||||||
|
through expandable help, tooltips, side panels, or review steps.
|
||||||
|
- Explain blocked actions in actionable language. A disabled control or failed
|
||||||
|
step should say what is missing, who can fix it, and where to go, for example
|
||||||
|
"A system administrator must allow this provider" or "Configure the provider
|
||||||
|
in Settings > File Providers before linking a folder here."
|
||||||
|
- Reuse visual language and placements consistently. Similar configuration,
|
||||||
|
policy, connection, credential, review, and confirmation flows should share
|
||||||
|
components, button placement, modal behavior, problem lists, and empty/error
|
||||||
|
states.
|
||||||
|
- Use modals and step flows for focused creation/editing where they reduce page
|
||||||
|
clutter. Reserve large always-open pages for overview, comparison, and
|
||||||
|
repeated administration work.
|
||||||
|
- Treat diagnostics as product UX. Validation results, preflight blockers,
|
||||||
|
policy explanations, permission denials, and missing capabilities should be
|
||||||
|
understandable to a non-technical operator before exposing internal details.
|
||||||
|
|
||||||
|
This is a product quality gate. New admin/configuration surfaces should not be
|
||||||
|
considered complete if they expose all options at once, require JSON editing,
|
||||||
|
hide why an action is unavailable, or use a one-off layout where a shared
|
||||||
|
pattern exists.
|
||||||
|
|
||||||
|
## Focus Rules
|
||||||
|
|
||||||
|
1. Build one reference journey per wave.
|
||||||
|
2. Do not implement a module because the repository exists.
|
||||||
|
3. Do not add module-to-module imports for optional behavior.
|
||||||
|
4. Every new domain module must justify its own semantics beyond `cases`,
|
||||||
|
`workflow`, `tasks`, `forms`, and `files`.
|
||||||
|
5. Prefer connector first when an external specialist system is likely to remain
|
||||||
|
the system of record.
|
||||||
|
6. Keep active work in Gitea issues; keep durable context in docs and synced
|
||||||
|
wiki pages.
|
||||||
|
7. A module moves from scaffold to implementation only when it has an owner,
|
||||||
|
reference journey, boundary notes, capability contracts, and testable MVP.
|
||||||
|
|
||||||
|
## Capability Map
|
||||||
|
|
||||||
|
| Capability | Likely owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Public application entry point | `govoplan-portal` |
|
||||||
|
| Structured forms and validation | `govoplan-forms` |
|
||||||
|
| Uploaded files and managed storage | `govoplan-files` |
|
||||||
|
| Case record and lifecycle | `govoplan-cases` |
|
||||||
|
| Workflow transitions and automation | `govoplan-workflow` |
|
||||||
|
| Action/effect catalogue and automation runner | first `govoplan-workflow`; possible future `govoplan-automation` if it outgrows workflow |
|
||||||
|
| Internal work queues and tasks | `govoplan-tasks` |
|
||||||
|
| Appointment proposals and booking | `govoplan-appointments`, `govoplan-calendar` |
|
||||||
|
| Postbox, email, and notifications | `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications` |
|
||||||
|
| Identity trust, device keys, and encrypted postbox key contracts | `govoplan-identity-trust`, `govoplan-access`, `govoplan-postbox` |
|
||||||
|
| Service directory/catalog | `govoplan-portal` |
|
||||||
|
| Permit/document generation | `govoplan-templates`, `govoplan-dms` |
|
||||||
|
| Payment capture and accounting handoff | `govoplan-payments`, `govoplan-ledger` |
|
||||||
|
| Roles, permissions, tenants, policy, audit | `govoplan-access`, `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` |
|
||||||
|
| External software integration | `govoplan-connectors` |
|
||||||
|
| Recurring extraction and transformation | possible future `govoplan-datasources`, possible future `govoplan-dataflow` |
|
||||||
|
| Reports, BI, and management visibility | `govoplan-reporting` |
|
||||||
|
|
||||||
|
## Configuration And Safety Target
|
||||||
|
|
||||||
|
The long-term target is that operators configure the platform through the UI
|
||||||
|
instead of editing files for normal operation.
|
||||||
|
|
||||||
|
UI-managed configuration should include:
|
||||||
|
|
||||||
|
- module installation, enablement, lifecycle state, and health
|
||||||
|
- tenants, users, groups, roles, policies, and permissions
|
||||||
|
- connectors, credentials, secret references, and external service tests
|
||||||
|
- workflows, forms, templates, task queues, schedules, and notifications
|
||||||
|
- configuration package import/export and environment-specific data collection
|
||||||
|
- retention, audit, privacy, maintenance mode, and safety controls
|
||||||
|
- deployment-visible settings such as public URLs, mail senders, storage
|
||||||
|
profiles, queues, and worker capabilities
|
||||||
|
|
||||||
|
Safety controls should include dry-run plans, field-level validation, policy
|
||||||
|
explanations, two-person approval for destructive changes, versioned
|
||||||
|
configuration history, rollback paths, audit events, and maintenance-mode
|
||||||
|
guards.
|
||||||
|
|
||||||
|
The initial safety metadata contract lives in
|
||||||
|
`govoplan_core.core.configuration_safety`. It classifies known configuration
|
||||||
|
fields as UI-managed or deployment-managed, assigns risk levels, marks secret
|
||||||
|
handling as reference-only or env-only, and declares dry-run, policy
|
||||||
|
explanation, audit, approval, rollback-history, maintenance-mode, and RBAC
|
||||||
|
requirements. Admin UI editors should consume this metadata before exposing
|
||||||
|
powerful settings.
|
||||||
|
|
||||||
|
The initial executable guardrail path is `plan_configuration_change(...)`,
|
||||||
|
exposed through:
|
||||||
|
|
||||||
|
- `GET /api/v1/admin/configuration-safety`
|
||||||
|
- `POST /api/v1/admin/configuration-safety/plan`
|
||||||
|
|
||||||
|
The planner reports missing scopes, dry-run requirements, maintenance-mode
|
||||||
|
requirements, two-person approval status, secret-reference violations,
|
||||||
|
rollback-history requirements, policy explanations, and audit event names before
|
||||||
|
an editor applies a high-impact configuration change.
|
||||||
|
|
||||||
|
## Reference Journeys
|
||||||
|
|
||||||
|
The roadmap should be driven by three journeys.
|
||||||
|
|
||||||
|
### Journey 1: Permit To Payment
|
||||||
|
|
||||||
|
This is the primary public-administration journey.
|
||||||
|
|
||||||
|
1. A person applies for a permit through the public portal.
|
||||||
|
2. The applicant uploads required files and submits structured form data.
|
||||||
|
3. Submission creates a case, a workflow instance, and an internal task.
|
||||||
|
4. Completing the task creates a postbox message, a notification, and an email
|
||||||
|
notification with an appointment proposal.
|
||||||
|
5. The applicant accepts an appointment, which updates the calendar and the
|
||||||
|
workflow state.
|
||||||
|
6. During the appointment, the case is opened and the permit is generated from
|
||||||
|
a governed template.
|
||||||
|
7. The payment is processed and linked to the case and accounting handoff.
|
||||||
|
8. The permit, payment evidence, communication history, audit trail, retention
|
||||||
|
state, and records evidence remain available according to policy.
|
||||||
|
|
||||||
|
This journey proves the platform can coordinate modules without core knowing
|
||||||
|
module internals.
|
||||||
|
|
||||||
|
### Journey 2: Training To Certificate
|
||||||
|
|
||||||
|
This is the best university-administration and internal-administration journey.
|
||||||
|
|
||||||
|
1. Course or training offer is planned.
|
||||||
|
2. Room, trainer, resource, and capacity are booked.
|
||||||
|
3. Participants register or are assigned.
|
||||||
|
4. Attendance is tracked.
|
||||||
|
5. Certificate or participation confirmation is issued.
|
||||||
|
6. Evidence remains available through records, files, audit, and docs.
|
||||||
|
|
||||||
|
This journey keeps `booking`, `resources`, `learning`, and `certificates`
|
||||||
|
focused instead of becoming broad ERP replacements.
|
||||||
|
|
||||||
|
### Journey 3: Report To Resolution
|
||||||
|
|
||||||
|
This is the internal operations and municipal issue-reporting journey.
|
||||||
|
|
||||||
|
1. A person reports an issue.
|
||||||
|
2. The issue is triaged into helpdesk, facilities, assets, or a case.
|
||||||
|
3. Work is assigned, tracked, and escalated.
|
||||||
|
4. Evidence, communication, and status updates are preserved.
|
||||||
|
5. Reports show workload, SLA, recurring problems, and completion.
|
||||||
|
|
||||||
|
This journey prevents `helpdesk`, `issue-reporting`, `facilities`, and `assets`
|
||||||
|
from becoming disconnected ticket silos.
|
||||||
|
|
||||||
|
## Roadmap Waves
|
||||||
|
|
||||||
|
### Wave 0: Platform Spine
|
||||||
|
|
||||||
|
Goal: make the platform safe to configure and extend.
|
||||||
|
|
||||||
|
Refine:
|
||||||
|
|
||||||
|
- `govoplan-core`: module discovery, capabilities, events, migrations, release
|
||||||
|
catalog, configuration package runtime, WebUI shell.
|
||||||
|
- `govoplan-access`: identities, sessions, API keys, users, groups, roles,
|
||||||
|
memberships, function assignments, delegation, RBAC decisions.
|
||||||
|
- `govoplan-tenancy`: tenant and organizational-unit boundaries.
|
||||||
|
- `govoplan-identity-trust`: initial trust contracts for device keys, public key
|
||||||
|
directory, assurance, and later encrypted postbox key access.
|
||||||
|
- `govoplan-policy`: policy sources, policy decisions, retention inputs.
|
||||||
|
- `govoplan-audit`: audit sink, audit routes, trace context, retention
|
||||||
|
cooperation.
|
||||||
|
- `govoplan-admin`: UI-managed configuration with preflight, rollback, audit,
|
||||||
|
and approval controls.
|
||||||
|
- `govoplan-dashboard`: configurable user home assembled from module-provided
|
||||||
|
widgets, with core providing only a minimal fallback when the module is absent.
|
||||||
|
- `govoplan-docs`: configured-system documentation and evidence-aware help.
|
||||||
|
- `govoplan-ops`: deployment profiles, health checks, worker split, sizing
|
||||||
|
assumptions.
|
||||||
|
|
||||||
|
Do not expand domain scope in this wave. The output is a dependable platform
|
||||||
|
surface for later modules.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- module enablement and capability lookup are stable
|
||||||
|
- configuration package preflight works for at least one simple package
|
||||||
|
- audit and policy decisions are visible in admin flows
|
||||||
|
- access distinguishes identity, account, function, role, and right in durable
|
||||||
|
contracts
|
||||||
|
- action/effect automation contracts are specified before hidden side effects
|
||||||
|
spread across modules
|
||||||
|
- docs can show installed/enabled modules and configured routes
|
||||||
|
|
||||||
|
### Wave 1: Permit-To-Payment MVP
|
||||||
|
|
||||||
|
Goal: one complete public-administration process.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-forms-runtime`: form submissions, drafts, validation, attachments,
|
||||||
|
and submission evidence.
|
||||||
|
2. `govoplan-portal`: public authenticated and unauthenticated entry points.
|
||||||
|
3. `govoplan-files`: upload, evidence links, file spaces, and permissioned
|
||||||
|
access.
|
||||||
|
4. `govoplan-cases`: case record, status, assignments, deadlines, and case
|
||||||
|
evidence.
|
||||||
|
5. `govoplan-workflow`: state machine, transitions, commands, and module
|
||||||
|
handoff.
|
||||||
|
6. `govoplan-tasks`: work queues, assignments, due dates, and follow-ups.
|
||||||
|
7. `govoplan-templates`: permit/decision document generation.
|
||||||
|
8. `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications`: applicant and
|
||||||
|
internal communication, with the postbox model compatible with later E2EE
|
||||||
|
and role-bound access.
|
||||||
|
9. `govoplan-calendar`, `govoplan-appointments`, `govoplan-booking`: appointment
|
||||||
|
or booking handoff for the reference process.
|
||||||
|
10. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment
|
||||||
|
capture, payment evidence, and accounting/e-invoice handoff.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- a permit-to-payment package can be installed in a local demo
|
||||||
|
- every step has audit evidence
|
||||||
|
- every module integration uses capabilities, events, or DTOs
|
||||||
|
- user-facing documentation describes only the configured process
|
||||||
|
|
||||||
|
### Wave 2: Booking And Resource Operations
|
||||||
|
|
||||||
|
Goal: make time, capacity, and resource allocation a reusable product area.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-booking`: booking rules, capacity, quotas, waitlists, cancellation,
|
||||||
|
attendance, no-shows, and approvals.
|
||||||
|
2. `govoplan-resources`: rooms, equipment, vehicles, counters, labs, capacity,
|
||||||
|
availability, and maintenance blocks.
|
||||||
|
3. `govoplan-calendar`: event and availability integration.
|
||||||
|
4. `govoplan-appointments`: appointment-specific booking surfaces.
|
||||||
|
5. `govoplan-facilities`: buildings, rooms, maintenance, access zones,
|
||||||
|
inspections, and defects.
|
||||||
|
6. `govoplan-assets`: inventory, assignment, lifecycle, maintenance, and handover
|
||||||
|
evidence.
|
||||||
|
|
||||||
|
Reference journey: reserve a room/resource for a training or appointment and
|
||||||
|
preserve all booking evidence.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- bookable resources are not hardcoded into appointments
|
||||||
|
- booking decisions are explainable and auditable
|
||||||
|
- resources can be blocked by maintenance or facility status
|
||||||
|
|
||||||
|
### Wave 3: Training And Certificates
|
||||||
|
|
||||||
|
Goal: support university-style and internal public-sector training without
|
||||||
|
building a full learning-management system first.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-learning`: course planning, course booking, participant lists,
|
||||||
|
trainers, attendance, evaluations, and learning records.
|
||||||
|
2. `govoplan-certificates`: participation confirmations, certificates,
|
||||||
|
verification, revocation, and evidence links.
|
||||||
|
3. `govoplan-booking`, `govoplan-resources`, `govoplan-calendar`: reuse booking
|
||||||
|
and room/resource allocation.
|
||||||
|
4. `govoplan-templates`, `govoplan-files`, `govoplan-records`: certificate
|
||||||
|
generation and durable retention.
|
||||||
|
|
||||||
|
Reference journey: plan a course, book resources, register participants, track
|
||||||
|
attendance, and issue a certificate.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- attendance can produce certificate eligibility
|
||||||
|
- certificates can be verified later
|
||||||
|
- course operations do not depend on university-specific assumptions
|
||||||
|
|
||||||
|
### Wave 4: Report-To-Resolution Operations
|
||||||
|
|
||||||
|
Goal: cover internal support and public issue reporting.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-issue-reporting`: public/internal reports, categories, intake,
|
||||||
|
location, evidence, and triage.
|
||||||
|
2. `govoplan-helpdesk`: service desk tickets, queues, SLAs, assignments,
|
||||||
|
escalation, and resolution evidence.
|
||||||
|
3. `govoplan-facilities` and `govoplan-assets`: issue handoff to maintenance or
|
||||||
|
asset lifecycle.
|
||||||
|
4. `govoplan-cases`, `govoplan-workflow`, `govoplan-tasks`: escalation into
|
||||||
|
formal administrative matters.
|
||||||
|
5. `govoplan-reporting`: workload, SLA, recurring defects, and status reports.
|
||||||
|
|
||||||
|
Reference journey: report a facility issue, triage it, assign work, resolve it,
|
||||||
|
and report recurring defects.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- issue reporting is not just a generic form inbox
|
||||||
|
- helpdesk tickets can remain lightweight unless formal case handling is needed
|
||||||
|
- operational metrics exist without custom SQL
|
||||||
|
|
||||||
|
### Wave 5: Records, DMS, Transparency
|
||||||
|
|
||||||
|
Goal: make evidence legally and organizationally durable.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-dms`: document lifecycle, collaboration, versions, locks, approvals,
|
||||||
|
and DMS connectors.
|
||||||
|
2. `govoplan-records`: file plans, classification, retention schedules, disposal
|
||||||
|
holds, archive handoff, and records evidence.
|
||||||
|
3. `govoplan-policy` and `govoplan-audit`: retention policy, legal hold, audit
|
||||||
|
traceability.
|
||||||
|
4. `govoplan-search`: permissioned cross-module discovery.
|
||||||
|
5. `govoplan-transparency`: FOI/public-records requests, redaction, publication,
|
||||||
|
and disclosure evidence.
|
||||||
|
|
||||||
|
Reference journey: close a case, classify records, apply retention, and answer a
|
||||||
|
transparency request.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- files, documents, and records are not treated as the same concept
|
||||||
|
- transparency disclosure can redact and explain evidence
|
||||||
|
- retention and archive handoff are policy-governed
|
||||||
|
|
||||||
|
### Wave 6: Finance, Procurement, Contracts, Grants
|
||||||
|
|
||||||
|
Goal: cover the back-office procedures around spending, obligations, funding,
|
||||||
|
and revenue without replacing a finance system too early.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-procurement`: purchase requests, approvals, vendor comparison,
|
||||||
|
tender references, goods receipt, and handoff.
|
||||||
|
2. `govoplan-contracts`: contract register, obligations, renewals, reminders,
|
||||||
|
and responsible units.
|
||||||
|
3. `govoplan-grants`: funding programs, applications, eligibility, awards,
|
||||||
|
milestones, claims, and reporting duties.
|
||||||
|
4. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment,
|
||||||
|
accounting, and e-invoice handoff.
|
||||||
|
5. `govoplan-erp`: integration with the actual system of record, not a full ERP
|
||||||
|
replacement unless later justified.
|
||||||
|
|
||||||
|
Reference journey: purchase or grant approval through obligation tracking and
|
||||||
|
financial handoff.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- ERP remains an integration point unless GovOPlaN must own semantics
|
||||||
|
- contracts and grants produce obligations, deadlines, and reporting tasks
|
||||||
|
- financial evidence links back to cases, files, and audit
|
||||||
|
|
||||||
|
### Wave 7: Institutional Governance And Participation
|
||||||
|
|
||||||
|
Goal: support public bodies, municipalities, ministries, and universities in
|
||||||
|
formal coordination.
|
||||||
|
|
||||||
|
Create or refine in this order:
|
||||||
|
|
||||||
|
1. `govoplan-committee`: committees, boards, councils, agendas, minutes,
|
||||||
|
decisions, voting, and follow-up tasks.
|
||||||
|
2. `govoplan-consultation`: hearings, public consultations, stakeholder
|
||||||
|
feedback, formal comments, response matrices, and publication workflows.
|
||||||
|
3. `govoplan-campaign`: communication campaigns, recipients, templates, review,
|
||||||
|
sending control, and reports.
|
||||||
|
4. `govoplan-addresses`: persons, organizations, contacts, distribution lists,
|
||||||
|
and recipient import/export.
|
||||||
|
5. `govoplan-portal` and `govoplan-transparency`: public participation and
|
||||||
|
publication surfaces.
|
||||||
|
|
||||||
|
Reference journey: prepare a committee decision, publish consultation material,
|
||||||
|
collect feedback, document the decision, and assign follow-up tasks.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- decisions create durable tasks and records
|
||||||
|
- consultations can publish evidence without exposing restricted material
|
||||||
|
- campaign and address behavior stays optional and capability-based
|
||||||
|
|
||||||
|
### Wave 8: Integration, Data, And Reporting
|
||||||
|
|
||||||
|
Goal: connect the platform to real institutional landscapes.
|
||||||
|
|
||||||
|
Refine:
|
||||||
|
|
||||||
|
- `govoplan-connectors`: integration catalog, connector metadata, adapter
|
||||||
|
lifecycle, test harnesses.
|
||||||
|
- `govoplan-idm`, `govoplan-identity-trust`: identity provider, directory, and
|
||||||
|
assurance integration.
|
||||||
|
- `govoplan-fit-connect`, `govoplan-xoev`, `govoplan-xta-osci`: public-sector
|
||||||
|
transport and standards integration.
|
||||||
|
- `govoplan-reporting`: operational reporting, scheduled outputs, exports, and
|
||||||
|
dashboard data.
|
||||||
|
- `govoplan-search`: permissioned cross-module discovery.
|
||||||
|
|
||||||
|
Create only when justified:
|
||||||
|
|
||||||
|
- `govoplan-datasources`: source catalog, connection profiles, schema discovery,
|
||||||
|
freshness, provenance.
|
||||||
|
- `govoplan-dataflow`: transformations, validation, lineage, scheduled runs,
|
||||||
|
publication outputs.
|
||||||
|
- `govoplan-projects`: only if OpenProject connectors and existing cases/tasks
|
||||||
|
cannot cover the required semantics.
|
||||||
|
|
||||||
|
Reference journey: monthly data extraction, transformation, validation, approval,
|
||||||
|
publication, and reporting.
|
||||||
|
|
||||||
|
Recurring extraction/transformation should start as a configuration package
|
||||||
|
across connectors, files, workflow, reporting, and templates. The package should
|
||||||
|
register sources, declare schemas, define mapping/validation versions, schedule
|
||||||
|
runs, produce previewable diffs, write governed outputs, and preserve lineage,
|
||||||
|
hashes, operator actions, and audit evidence. Create `govoplan-datasources` or
|
||||||
|
`govoplan-dataflow` only after this work exposes repeated contracts that do not
|
||||||
|
belong to existing modules.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- connector catalog exists before building many adapters
|
||||||
|
- dataflow is created only after recurring transformation becomes product
|
||||||
|
behavior
|
||||||
|
- reporting consumes governed sources with provenance
|
||||||
|
|
||||||
|
## Implementation Gates
|
||||||
|
|
||||||
|
Before a module receives implementation work, it needs:
|
||||||
|
|
||||||
|
- one reference journey step it owns
|
||||||
|
- ownership and boundary notes
|
||||||
|
- initial manifest and permission list
|
||||||
|
- capability contracts or API DTOs
|
||||||
|
- audit and policy expectations
|
||||||
|
- Gitea issues for MVP tasks
|
||||||
|
- docs page that can sync to the wiki
|
||||||
|
- focused tests that can run from the core environment
|
||||||
|
|
||||||
|
Before a module becomes release-included, it needs:
|
||||||
|
|
||||||
|
- installable Python package metadata where applicable
|
||||||
|
- WebUI package metadata where applicable
|
||||||
|
- module manifest entry point
|
||||||
|
- release catalog entry
|
||||||
|
- smoke test or permutation test
|
||||||
|
- no required imports from optional modules
|
||||||
|
|
||||||
|
## Priority Order Summary
|
||||||
|
|
||||||
|
1. Stabilize the platform spine.
|
||||||
|
2. Deliver permit-to-payment MVP.
|
||||||
|
3. Build booking and resource operations.
|
||||||
|
4. Add learning and certificates.
|
||||||
|
5. Add issue reporting and helpdesk.
|
||||||
|
6. Add records, DMS, search, and transparency.
|
||||||
|
7. Add procurement, contracts, grants, and finance handoff.
|
||||||
|
8. Add committee and consultation workflows.
|
||||||
|
9. Expand integration, dataflow, reporting, and operations.
|
||||||
|
|
||||||
|
## Deliberate Deferrals
|
||||||
|
|
||||||
|
Defer these until a reference journey proves the need:
|
||||||
|
|
||||||
|
- full ERP replacement
|
||||||
|
- native project management beyond connector support
|
||||||
|
- broad BI/dataflow platform
|
||||||
|
- every possible public-sector protocol adapter
|
||||||
|
- rich LMS behavior beyond training administration
|
||||||
|
- full qualified digital signing/trust services beyond the identity-trust and
|
||||||
|
encrypted-postbox contracts needed for early architecture safety
|
||||||
|
- mobile apps
|
||||||
|
- AI assistants embedded into workflows
|
||||||
|
|
||||||
|
These may become important, but they should not distract from the first complete
|
||||||
|
administrative journeys.
|
||||||
|
|
||||||
|
## Module And Integration Routing
|
||||||
|
|
||||||
|
This table maps current module and integration ideas to existing GovOPlaN
|
||||||
|
repositories or to explicit missing-module decisions.
|
||||||
|
|
||||||
|
| Idea | Owner | Tracking |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Government operations backbone reference model | `govoplan-core` | `add-ideas/govoplan-core#213` |
|
||||||
|
| Permit-to-payment configuration package | `govoplan-core` plus participating modules | `add-ideas/govoplan-core#214` |
|
||||||
|
| Fully UI-managed configuration with safety controls | `govoplan-admin`, `govoplan-core`, `govoplan-policy`, `govoplan-access`, `govoplan-audit` | `add-ideas/govoplan-core#218` |
|
||||||
|
| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` |
|
||||||
|
| Interface ethics and decision-consequence doctrine | `govoplan-core` plus all UI-owning modules | `add-ideas/govoplan-core#227` |
|
||||||
|
| Action/effect automation layer | first `govoplan-workflow`; possible future `govoplan-automation` | `add-ideas/govoplan-workflow#1` |
|
||||||
|
| E2EE role/function postbox architecture | `govoplan-postbox`, `govoplan-identity-trust`, `govoplan-access`, `govoplan-policy`, `govoplan-audit` | `add-ideas/govoplan-postbox#15`, `add-ideas/govoplan-identity-trust#1` |
|
||||||
|
| Identity, account, function, role, right semantic model | `govoplan-access` | `add-ideas/govoplan-access#9` |
|
||||||
|
| Role-based service directory/catalog | `govoplan-portal` | `add-ideas/govoplan-portal#1` |
|
||||||
|
| Unified inbox across tasks, postbox, notifications, and portal | `govoplan-core` coordination plus owning modules | `add-ideas/govoplan-tasks#1`, `add-ideas/govoplan-notifications#1` |
|
||||||
|
| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` |
|
||||||
|
| Native project-management module decision | connector-first through `govoplan-connectors`; no native project module yet | `add-ideas/govoplan-core#196`, `add-ideas/govoplan-connectors#1` |
|
||||||
|
| Datasources for databases, CSV, files, APIs | no repository yet; start with connectors/files/reporting and create `govoplan-datasources` only after the first package proves shared source-catalog ownership | `add-ideas/govoplan-core#197` |
|
||||||
|
| Dataflow for pipelines, BI, publication | no repository yet; start with workflow/reporting/connectors and create `govoplan-dataflow` only after repeated pipeline/lineage contracts emerge | `add-ideas/govoplan-core#198` |
|
||||||
|
| Monthly datasource and transformation workflows | first as configuration package across connectors, files, workflow, reporting, and templates | `add-ideas/govoplan-core#216` |
|
||||||
|
| Templates for letters, emails, forms, reports | `govoplan-templates`, separate from reporting | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-templates#1` |
|
||||||
|
| Reporting and BI | `govoplan-reporting`, separate from templates | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-reporting#1` |
|
||||||
|
| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` |
|
||||||
|
| Public-sector software integration catalogue | `govoplan-connectors` with core strategy index | `add-ideas/govoplan-core#191`, `add-ideas/govoplan-connectors#2` |
|
||||||
|
| Public-sector integration landscape catalogue | `govoplan-connectors` with core tracking | `add-ideas/govoplan-core#215` |
|
||||||
|
| Cases module concept | `govoplan-cases` | `add-ideas/govoplan-core#174` |
|
||||||
|
| Workflow module concept | `govoplan-workflow` | `add-ideas/govoplan-core#175` |
|
||||||
|
| Connectors module concept | `govoplan-connectors` | `add-ideas/govoplan-core#176` |
|
||||||
|
| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` |
|
||||||
|
| Consume sources and become a governed source | `govoplan-connectors` plus possible future `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` |
|
||||||
|
| Governed connector configuration, dry-run, and simulation runtime | `govoplan-connectors` | `add-ideas/govoplan-connectors#6` |
|
||||||
|
| Terminfindung and meeting scheduling polls | `govoplan-scheduling`; calendar primitives remain in calendar | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-scheduling#1` |
|
||||||
|
| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-calendar#1` |
|
||||||
|
| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-appointments#1` |
|
||||||
|
| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` |
|
||||||
|
| Forms | `govoplan-forms` for definitions and `govoplan-forms-runtime` for submissions/runtime behavior | `add-ideas/govoplan-core#194`, `add-ideas/govoplan-forms#1` |
|
||||||
|
| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` |
|
||||||
|
| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` |
|
||||||
|
| OpenDesk stack integration map | integration profile across IDM/access, mail/calendar, files/DMS, and connectors; not a monolithic module | `add-ideas/govoplan-core#195`, `add-ideas/govoplan-connectors#5` |
|
||||||
|
| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` |
|
||||||
|
| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` |
|
||||||
|
| Scalability profiles and autoscaling readiness | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#217` |
|
||||||
|
| Hardware sizing matrix and requirements calculator | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#219` |
|
||||||
|
| Collaboration suite integration strategy | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow`, `govoplan-tasks`, `govoplan-appointments`, `govoplan-calendar` | `add-ideas/govoplan-core#220` |
|
||||||
|
| Install/runtime configuration contract | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#19` |
|
||||||
|
| Installer/deployment operator flow | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#26` |
|
||||||
|
| Production-like deployment documentation | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#28` |
|
||||||
|
|
||||||
|
Boundary rationale lives in `MODULE_ARCHITECTURE.md`. Current decisions:
|
||||||
|
|
||||||
|
- templates and reporting are separate modules
|
||||||
|
- RSS/source consume-publish starts in connectors; datasources/dataflow are not
|
||||||
|
repositories yet
|
||||||
|
- calendar, scheduling, and appointments are three separate modules
|
||||||
|
- forms definitions and forms runtime are separate responsibilities
|
||||||
|
- OpenDesk is an integration profile across modules, not a monolithic module
|
||||||
|
- OpenProject is connector-first; no native projects module yet
|
||||||
|
- public-sector integration strategy stays in core; executable catalogue work
|
||||||
|
lives in connectors
|
||||||
|
- encrypted postbox and identity-trust are strategic contracts, not mail-module
|
||||||
|
behavior
|
||||||
|
- automation starts as workflow-owned action/effect execution and may split into
|
||||||
|
a dedicated module only after the runner becomes broader than workflow
|
||||||
|
|
||||||
|
The following modules are intentionally not created yet:
|
||||||
|
|
||||||
|
- `govoplan-datasources`
|
||||||
|
- `govoplan-dataflow`
|
||||||
|
- `govoplan-projects`
|
||||||
|
|
||||||
|
Create a repository only after a concrete implementation package proves that
|
||||||
|
existing connector, files, reporting, workflow, or task ownership is too narrow.
|
||||||
|
|
||||||
|
Core keeps the strategy index in
|
||||||
|
`PUBLIC_SECTOR_INTEGRATION_STRATEGY.md`: integration postures, default
|
||||||
|
ownership, and prioritization rules. `govoplan-connectors` owns the detailed
|
||||||
|
target inventory and connector entry shape in
|
||||||
|
`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`.
|
||||||
|
|
||||||
|
Release composition and tag-only repository handling are documented in
|
||||||
|
`RELEASE_DEPENDENCIES.md`.
|
||||||
|
|
||||||
|
## Next Practical Work
|
||||||
|
|
||||||
|
The next planning step should create or update Gitea issues for Wave 0 and Wave
|
||||||
|
1 only. Later waves should stay as roadmap context until the permit-to-payment
|
||||||
|
MVP is demonstrable.
|
||||||
|
|
||||||
|
Recommended immediate issue buckets:
|
||||||
|
|
||||||
|
- platform spine hardening
|
||||||
|
- configuration package preflight and rollback
|
||||||
|
- forms-runtime MVP
|
||||||
|
- portal submission MVP
|
||||||
|
- cases/workflow/tasks integration MVP
|
||||||
|
- template-generated decision document
|
||||||
|
- postbox/notification handoff
|
||||||
|
- appointment/booking handoff
|
||||||
|
- payment evidence handoff
|
||||||
|
- configured documentation for the reference process
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
# GovOPlaN Module And Integration Roadmap
|
|
||||||
|
|
||||||
This page maps current module and integration ideas to existing GovOPlaN repositories or to missing-module decisions. Issues are the active backlog. This document is durable routing context and should be mirrored to the Gitea wiki.
|
|
||||||
|
|
||||||
## Current Routing
|
|
||||||
|
|
||||||
| Idea | Owner | Tracking |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` |
|
|
||||||
| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` |
|
|
||||||
| Native project-management module decision | `govoplan-core` | `add-ideas/govoplan-core#196` |
|
|
||||||
| Datasources for databases, CSV, files, APIs | proposed `govoplan-datasources` | `add-ideas/govoplan-core#197` |
|
|
||||||
| Dataflow for pipelines, BI, publication | proposed `govoplan-dataflow` | `add-ideas/govoplan-core#198` |
|
|
||||||
| Templates for letters, emails, forms, reports | `govoplan-templates` | `add-ideas/govoplan-templates#1` |
|
|
||||||
| Reporting and BI | `govoplan-reporting` | `add-ideas/govoplan-reporting#1` |
|
|
||||||
| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` |
|
|
||||||
| Public-sector software integration catalogue | `govoplan-connectors` | `add-ideas/govoplan-connectors#2` |
|
|
||||||
| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` |
|
|
||||||
| Consume sources and become a governed source | `govoplan-connectors` plus proposed `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` |
|
|
||||||
| Terminfindung and meeting scheduling polls | `govoplan-scheduling` | local scaffold; remote creation tracked by `add-ideas/govoplan-core#199` |
|
|
||||||
| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-calendar#1` |
|
|
||||||
| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-appointments#1` |
|
|
||||||
| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` |
|
|
||||||
| Forms | `govoplan-forms` | `add-ideas/govoplan-forms#1` |
|
|
||||||
| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` |
|
|
||||||
| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` |
|
|
||||||
| OpenDesk stack integration map | `govoplan-connectors` | `add-ideas/govoplan-connectors#5` |
|
|
||||||
| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` |
|
|
||||||
| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` |
|
|
||||||
|
|
||||||
## Proposed Missing Modules
|
|
||||||
|
|
||||||
`govoplan-datasources` should exist only if source catalog ownership becomes broad enough to justify a module separate from connectors and reporting. Candidate responsibilities:
|
|
||||||
|
|
||||||
- source catalogue for SQL databases, CSV/Excel files, uploaded files, APIs, RSS feeds, and governed file locations
|
|
||||||
- credentials and connection profiles
|
|
||||||
- schema discovery and refresh cadence
|
|
||||||
- provenance, freshness, permission boundaries, and audit events
|
|
||||||
|
|
||||||
`govoplan-dataflow` should exist only if pipelines and publication become first-class product behavior. Candidate responsibilities:
|
|
||||||
|
|
||||||
- ingestion, transformation, validation, scheduling, and lineage
|
|
||||||
- connecting datasources to reports, APIs, RSS, exports, and downstream systems
|
|
||||||
- the "consume sources, become source" lifecycle
|
|
||||||
- publication-state and audit integration
|
|
||||||
|
|
||||||
`govoplan-projects` is undecided. The default path should be an OpenProject connector first. A native module is justified only if GovOPlaN must own project semantics beyond cases, tasks, workflow, appointments, documents, and reporting.
|
|
||||||
|
|
||||||
## Boundary Notes
|
|
||||||
|
|
||||||
- `govoplan-connectors` owns protocol and external-system integration strategy. It should not own business semantics once a domain module exists.
|
|
||||||
- `govoplan-files` owns file storage semantics and file-provider contracts. Remote provider implementations must stay optional.
|
|
||||||
- `govoplan-dms` owns document lifecycle, collaboration, versions, approvals, locks, retention, and legal hold.
|
|
||||||
- `govoplan-addresses` owns persons, organizations, postal/email addresses, distribution lists, and recipient import/export.
|
|
||||||
- `govoplan-calendar` owns events, availability, resources, recurrence, and groupware calendar adapters.
|
|
||||||
- `govoplan-scheduling` owns meeting scheduling polls, participant availability collection, candidate-slot ranking, and decision handoff.
|
|
||||||
- `govoplan-appointments` owns public/internal appointment-booking workflows.
|
|
||||||
- `govoplan-idm` owns directory, provisioning, and identity-provider integration; `govoplan-access` consumes resolved principals and permissions.
|
|
||||||
- `govoplan-reporting` owns report definitions, BI views, scheduled outputs, and export targets.
|
|
||||||
- `govoplan-templates` owns reusable renderable templates, not data selection or persistence.
|
|
||||||
|
|
||||||
## Release Tooling Note
|
|
||||||
|
|
||||||
`scripts/push-release-tag.sh` covers the released full-product package set:
|
|
||||||
|
|
||||||
- `govoplan-access`
|
|
||||||
- `govoplan-admin`
|
|
||||||
- `govoplan-tenancy`
|
|
||||||
- `govoplan-policy`
|
|
||||||
- `govoplan-audit`
|
|
||||||
- `govoplan-files`
|
|
||||||
- `govoplan-mail`
|
|
||||||
- `govoplan-campaign`
|
|
||||||
- `govoplan-calendar`
|
|
||||||
- `govoplan-core`
|
|
||||||
|
|
||||||
It also includes existing roadmap/scaffold module repositories such as addresses, appointments, connectors, DMS, forms, IDM, reporting, scheduling, templates, workflow, XOE/V, and XRechnung as tag-only repositories. Tag-only repositories are committed, tagged, and pushed with the same release tag, but they are not added to `requirements-release.txt` or `webui/package.release.json` until they contain installable package metadata.
|
|
||||||
|
|
||||||
Proposed modules without repositories, such as `govoplan-datasources`, `govoplan-dataflow`, and possibly `govoplan-projects`, cannot be included until their repositories are created.
|
|
||||||
102
docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md
Normal file
102
docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
# GovOPlaN Interface Ethics And Design Doctrine
|
||||||
|
|
||||||
|
This document captures the product-level design doctrine for GovOPlaN. It is
|
||||||
|
more durable than an individual screen design and should guide admin,
|
||||||
|
configuration, workflow, policy, portal, and operational UI decisions.
|
||||||
|
|
||||||
|
GovOPlaN is meant to support administrative responsibility. The interface must
|
||||||
|
therefore make context, decision, consequence, and traceability visible enough
|
||||||
|
that users can act deliberately instead of being pushed through opaque
|
||||||
|
automation.
|
||||||
|
|
||||||
|
## Core Doctrine
|
||||||
|
|
||||||
|
1. Context, decision, and consequence belong together.
|
||||||
|
2. Decisions should not silently happen.
|
||||||
|
3. Transparency comes before convenience when rights, duties, records, money,
|
||||||
|
access, or legal effects are involved.
|
||||||
|
4. Explicit state is preferable to implicit state.
|
||||||
|
5. Navigation is not consent.
|
||||||
|
6. Responsibility cannot be delegated to the system.
|
||||||
|
7. Traceability is part of the action, not a later reporting feature.
|
||||||
|
8. Context loss is a product defect.
|
||||||
|
|
||||||
|
These rules do not mean every screen should become verbose. They mean the
|
||||||
|
interface must expose the right explanation at the moment of decision and keep
|
||||||
|
technical detail available without making it the default surface.
|
||||||
|
|
||||||
|
## Decision Surface Contract
|
||||||
|
|
||||||
|
Any action that changes records, rights, policies, retention, communication,
|
||||||
|
payments, external systems, or workflow state should answer these questions
|
||||||
|
before execution:
|
||||||
|
|
||||||
|
- What object, person, organization, or process is affected?
|
||||||
|
- Which authority or role allows the actor to do this?
|
||||||
|
- What will change immediately?
|
||||||
|
- What downstream effects may happen?
|
||||||
|
- Can the action be undone, superseded, or only corrected later?
|
||||||
|
- What evidence or audit entry will be created?
|
||||||
|
- Which policy, configuration, or missing capability blocks the action?
|
||||||
|
- Who can resolve a blocker?
|
||||||
|
|
||||||
|
The answer may be shown through inline labels, a review step, a side panel, a
|
||||||
|
problem list, or a confirmation dialog. The important point is that consequence
|
||||||
|
and responsibility are not hidden behind a generic submit button.
|
||||||
|
|
||||||
|
## Contestability
|
||||||
|
|
||||||
|
Administrative decisions are often contestable or reviewable. GovOPlaN should
|
||||||
|
therefore preserve the path from input to decision:
|
||||||
|
|
||||||
|
- source data and attachments
|
||||||
|
- workflow state and task assignment
|
||||||
|
- policy decisions and source path
|
||||||
|
- actor and delegation context
|
||||||
|
- generated document/template version
|
||||||
|
- external handoff result
|
||||||
|
- notification or postbox delivery evidence
|
||||||
|
- retention and record classification state
|
||||||
|
|
||||||
|
Where a user sees a decision, they should be able to reach the provenance that
|
||||||
|
explains how the system got there. This is especially important for denials,
|
||||||
|
locks, calculated defaults, generated documents, payment state, retention
|
||||||
|
state, and access decisions.
|
||||||
|
|
||||||
|
## Anti-Patterns
|
||||||
|
|
||||||
|
Avoid these patterns in GovOPlaN interfaces:
|
||||||
|
|
||||||
|
- Magical buttons that execute multiple side effects without preview.
|
||||||
|
- Process tunnels that hide where the user is in an administrative procedure.
|
||||||
|
- Silent automation that changes external systems without an audit-visible
|
||||||
|
command record.
|
||||||
|
- Friendly hiding that removes complexity at the cost of obscuring authority,
|
||||||
|
consequence, or accountability.
|
||||||
|
- Disabled controls without actionable explanation.
|
||||||
|
- Configuration screens that ask users to edit raw JSON as the normal path.
|
||||||
|
|
||||||
|
## Automation Rule
|
||||||
|
|
||||||
|
Automation must use the same governed action surface as a human actor. The
|
||||||
|
system may execute actions as a system actor, but it must still run through
|
||||||
|
policy checks, capability contracts, audit, idempotency, and failure handling.
|
||||||
|
|
||||||
|
When an automated decision is not clear, GovOPlaN should create a manual
|
||||||
|
exception, task, or review item instead of guessing silently.
|
||||||
|
|
||||||
|
## Relationship To UI Components
|
||||||
|
|
||||||
|
Shared components should make this doctrine easy to follow:
|
||||||
|
|
||||||
|
- preflight and problem-list components for blockers
|
||||||
|
- policy source path and effective decision displays
|
||||||
|
- action review panels for consequence preview
|
||||||
|
- audit/provenance links on decision outputs
|
||||||
|
- guided dialogs for risky configuration
|
||||||
|
- disabled-action explanations with actor and next step
|
||||||
|
- confirmation dialogs that distinguish reversible, corrective, and destructive
|
||||||
|
actions
|
||||||
|
|
||||||
|
The UI/UX decision ledger defines concrete implementation rules. This doctrine
|
||||||
|
defines why those rules exist.
|
||||||
@@ -2,10 +2,17 @@
|
|||||||
|
|
||||||
GovOPlaN is structured as a platform kernel plus installable modules. The kernel starts and composes the platform. Modules own product behavior and contribute backend routes, database metadata, permissions, WebUI routes, navigation metadata, capabilities, and events.
|
GovOPlaN is structured as a platform kernel plus installable modules. The kernel starts and composes the platform. Modules own product behavior and contribute backend routes, database metadata, permissions, WebUI routes, navigation metadata, capabilities, and events.
|
||||||
|
|
||||||
The current package name is still `govoplan-core`, but the architecture target is a smaller kernel. Access, tenancy, policy, audit, and admin semantics are platform-module responsibilities and should be extracted in stages.
|
The current package name is still `govoplan-core`, but the architecture target is a smaller kernel. Access, tenancy, policy, audit, and admin semantics are platform-module responsibilities.
|
||||||
|
|
||||||
The concrete access/auth/RBAC extraction path is tracked in
|
Access extraction is complete enough that current ownership is described here,
|
||||||
[`ACCESS_EXTRACTION_PLAN.md`](ACCESS_EXTRACTION_PLAN.md).
|
in [`ACCESS_RBAC_MODEL.md`](ACCESS_RBAC_MODEL.md), and in the
|
||||||
|
`govoplan-access` repository docs.
|
||||||
|
The event and audit trace contract is tracked in
|
||||||
|
[`EVENTS_AND_AUDIT.md`](EVENTS_AND_AUDIT.md).
|
||||||
|
Policy decision, source provenance, and explain-response contracts are tracked
|
||||||
|
in [`POLICY_CONTRACTS.md`](POLICY_CONTRACTS.md).
|
||||||
|
The experimental remote WebUI bundle loading design is tracked in
|
||||||
|
[`REMOTE_WEBUI_BUNDLES.md`](REMOTE_WEBUI_BUNDLES.md).
|
||||||
|
|
||||||
## Layer Model
|
## Layer Model
|
||||||
|
|
||||||
@@ -35,14 +42,15 @@ The kernel must not own product semantics such as users, tenants, RBAC decisions
|
|||||||
## Current Compatibility Responsibilities
|
## Current Compatibility Responsibilities
|
||||||
|
|
||||||
During the staged split, `govoplan-core` still contains compatibility surfaces
|
During the staged split, `govoplan-core` still contains compatibility surfaces
|
||||||
for access, auth, tenancy, RBAC, governance, audit, CSRF/API helpers, and
|
for tenancy settings, governance/policy contracts, audit helpers, CSRF/API
|
||||||
secret helpers. The extracted access implementation now lives in
|
helpers, and secret helpers. The extracted access implementation lives in
|
||||||
`govoplan-access`; live legacy ORM table definitions have been split across
|
`govoplan-access`; live ORM table definitions have been split across their
|
||||||
their platform owners while retaining historical table names. The old core
|
platform owners using module-prefixed table names. The old core route,
|
||||||
model, route, admin-service, and access-security import shims have been
|
admin-service, and access-security re-export modules have been removed.
|
||||||
removed; callers must use module-owned imports or kernel capabilities. The
|
Callers must use module-owned imports, the public `govoplan_access.auth` request
|
||||||
remaining compatibility surfaces are temporary until the matching platform
|
dependency API, or kernel capabilities.
|
||||||
modules are fully self-contained:
|
The remaining platform compatibility surfaces are temporary until the matching
|
||||||
|
platform modules are fully self-contained:
|
||||||
|
|
||||||
- `govoplan-access`
|
- `govoplan-access`
|
||||||
- `govoplan-tenancy`
|
- `govoplan-tenancy`
|
||||||
@@ -54,6 +62,18 @@ New code should avoid deepening these compatibility dependencies. Prefer explici
|
|||||||
|
|
||||||
Core must not import module feature pages or module business logic directly. It should interact with modules through manifests, entry points, metadata, capabilities, events, and route contributions.
|
Core must not import module feature pages or module business logic directly. It should interact with modules through manifests, entry points, metadata, capabilities, events, and route contributions.
|
||||||
|
|
||||||
|
The compatibility/deprecation plan for the current split line is:
|
||||||
|
|
||||||
|
- keep documented public compatibility imports until the owning module exposes a
|
||||||
|
stable replacement and all in-tree callers have migrated
|
||||||
|
- remove deep implementation re-export modules once callers can use module-owned
|
||||||
|
public APIs or kernel capabilities
|
||||||
|
- preserve migration/table compatibility for already-created development and
|
||||||
|
release databases
|
||||||
|
- document remaining compatibility surfaces here and in the owning module README
|
||||||
|
- reject new cross-module imports that bypass manifests, capabilities, events,
|
||||||
|
or public module APIs
|
||||||
|
|
||||||
## Stable Kernel Contracts
|
## Stable Kernel Contracts
|
||||||
|
|
||||||
The following contracts are the baseline API that modules can rely on:
|
The following contracts are the baseline API that modules can rely on:
|
||||||
@@ -71,9 +91,15 @@ The following contracts are the baseline API that modules can rely on:
|
|||||||
- WebUI module contribution contract
|
- WebUI module contribution contract
|
||||||
- navigation metadata contract
|
- navigation metadata contract
|
||||||
- command/event envelope contract
|
- command/event envelope contract
|
||||||
|
- policy decision and source provenance contract in `govoplan_core.core.policy`
|
||||||
|
|
||||||
Changes to these contracts must be versioned or accompanied by compatibility shims.
|
Changes to these contracts must be versioned or accompanied by compatibility shims.
|
||||||
|
|
||||||
|
This list is the Milestone A kernel-contract freeze baseline. New module work
|
||||||
|
may extend the kernel by adding explicit contracts, but existing contracts must
|
||||||
|
remain source-compatible through the 0.1.x split line unless a migration shim
|
||||||
|
and deprecation note are provided.
|
||||||
|
|
||||||
Known access-related capability names are defined in
|
Known access-related capability names are defined in
|
||||||
`govoplan_core.core.access`, including:
|
`govoplan_core.core.access`, including:
|
||||||
|
|
||||||
@@ -99,22 +125,23 @@ access/tenant ORM models when they need labels, group membership, default
|
|||||||
access provisioning, counts, audit actor labels, or tenant metadata.
|
access provisioning, counts, audit actor labels, or tenant metadata.
|
||||||
|
|
||||||
FastAPI route dependencies for authenticated endpoints are access-owned and
|
FastAPI route dependencies for authenticated endpoints are access-owned and
|
||||||
published from `govoplan_access.backend.auth.dependencies`. Routers may import
|
published from `govoplan_access.auth`. Routers may import that public API for
|
||||||
that dependency module directly until a more generic request-principal adapter
|
`ApiPrincipal`, `get_api_principal`, `has_scope`, `require_scope`, and
|
||||||
exists; they must not import access ORM models or other access implementation
|
`require_any_scope`; they must not import access ORM models or
|
||||||
internals.
|
`govoplan_access.backend.*` implementation internals.
|
||||||
|
|
||||||
Current live table ownership:
|
Current live table ownership:
|
||||||
|
|
||||||
- `govoplan-tenancy`: `tenants`
|
- `govoplan-tenancy`: `tenancy_tenants`
|
||||||
- `govoplan-access`: `accounts`, `users`, `groups`, `roles`,
|
- `govoplan-access`: `access_accounts`, `access_users`, `access_groups`,
|
||||||
`system_role_assignments`, `user_group_memberships`,
|
`access_roles`, `access_system_role_assignments`,
|
||||||
`user_role_assignments`, `group_role_assignments`, `api_keys`,
|
`access_user_group_memberships`, `access_user_role_assignments`,
|
||||||
`auth_sessions`
|
`access_group_role_assignments`, `access_api_keys`,
|
||||||
- `govoplan-admin`: `governance_templates`,
|
`access_auth_sessions`
|
||||||
`governance_template_assignments`
|
- `govoplan-admin`: `admin_governance_templates`,
|
||||||
|
`admin_governance_template_assignments`
|
||||||
- `govoplan-audit`: `audit_log`
|
- `govoplan-audit`: `audit_log`
|
||||||
- `govoplan-core`: `system_settings`
|
- `govoplan-core`: `core_system_settings`
|
||||||
|
|
||||||
Current admin route ownership follows the same boundary: access contributes
|
Current admin route ownership follows the same boundary: access contributes
|
||||||
users, groups, roles, system accounts/roles, auth, sessions, and API-key
|
users, groups, roles, system accounts/roles, auth, sessions, and API-key
|
||||||
@@ -143,6 +170,183 @@ campaign file-share access, and core retention can call campaign-owned cleanup
|
|||||||
logic without importing campaign ORM models. Keep these contracts small
|
logic without importing campaign ORM models. Keep these contracts small
|
||||||
DTO/protocol surfaces and register concrete behavior from the owning module.
|
DTO/protocol surfaces and register concrete behavior from the owning module.
|
||||||
|
|
||||||
|
## API Efficiency Contracts
|
||||||
|
|
||||||
|
GovOPlaN uses conditional GET and delta collections to reduce reload cost
|
||||||
|
without giving every module a custom synchronization format.
|
||||||
|
|
||||||
|
### Conditional GET
|
||||||
|
|
||||||
|
Core applies conditional GET handling centrally for successful JSON `GET`
|
||||||
|
responses:
|
||||||
|
|
||||||
|
- Responses receive a weak `ETag` based on the serialized JSON body.
|
||||||
|
- Responses are marked `Cache-Control: private, no-cache`.
|
||||||
|
- Responses vary by `Authorization`, `Cookie`, `X-API-Key`, and
|
||||||
|
`Accept-Language`.
|
||||||
|
- Matching `If-None-Match` requests return `304 Not Modified` without a body.
|
||||||
|
- Responses with `Set-Cookie`, `Content-Disposition`, `Content-Encoding`, a
|
||||||
|
non-JSON content type, a non-200 status, or `Cache-Control: no-store` are not
|
||||||
|
converted.
|
||||||
|
|
||||||
|
The WebUI `apiFetch` client keeps an in-memory conditional cache for reusable
|
||||||
|
safe requests. It sends `If-None-Match` after an endpoint has returned an ETag,
|
||||||
|
returns the cached payload on `304`, and clears the cache generation after
|
||||||
|
unsafe methods.
|
||||||
|
|
||||||
|
This avoids retransmitting unchanged snapshots. It does not identify which row
|
||||||
|
changed inside a collection.
|
||||||
|
|
||||||
|
### Delta Collections
|
||||||
|
|
||||||
|
Collection endpoints that can expose row-level changes should use the shared
|
||||||
|
delta contract instead of inventing module-specific formats.
|
||||||
|
|
||||||
|
Core provides `core_change_sequence` as the shared monotonic change sequence.
|
||||||
|
Modules record append-only entries in the same database transaction as the
|
||||||
|
resource write. Watermarks are encoded as `seq:<number>` and should be treated
|
||||||
|
as opaque by clients.
|
||||||
|
|
||||||
|
Backend shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"items": [],
|
||||||
|
"deleted": [],
|
||||||
|
"watermark": "opaque-next-watermark",
|
||||||
|
"has_more": false,
|
||||||
|
"full": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Fields:
|
||||||
|
|
||||||
|
- `items`: changed or current items since the requested watermark.
|
||||||
|
- `deleted`: deleted item markers with at least `id`, and optionally
|
||||||
|
`resource_type`, `revision`, and `deleted_at`.
|
||||||
|
- `watermark`: opaque value the client sends as `since` on the next request.
|
||||||
|
- `has_more`: true when the client should request the next page with the
|
||||||
|
returned watermark.
|
||||||
|
- `full`: true when the response is a full snapshot rather than an incremental
|
||||||
|
delta.
|
||||||
|
|
||||||
|
Section-level settings endpoints use the same contract but replace `items`
|
||||||
|
with:
|
||||||
|
|
||||||
|
- `item`: the full settings object when `full: true`.
|
||||||
|
- `sections`: a map of changed settings sections when `full: false`.
|
||||||
|
- `changed_sections`: ordered section identifiers the client can merge into its
|
||||||
|
local settings object.
|
||||||
|
|
||||||
|
Recommended query parameters:
|
||||||
|
|
||||||
|
- `since`: opaque previous watermark. If omitted or expired, return a full
|
||||||
|
snapshot with `full: true`.
|
||||||
|
- `limit`: maximum number of changed items plus deleted markers.
|
||||||
|
- `include_deleted`: whether deleted markers should be returned.
|
||||||
|
- `cursor`: opaque keyset cursor for table pages where offset shifts would make
|
||||||
|
row-level merging unsafe.
|
||||||
|
|
||||||
|
Modules should record changes with:
|
||||||
|
|
||||||
|
- `module_id`: the owning module, for example `files`.
|
||||||
|
- `collection`: the delta collection, for example `files.assets`.
|
||||||
|
- `resource_type`: stable row kind, for example `file` or `folder`.
|
||||||
|
- `resource_id`: stable resource identifier.
|
||||||
|
- `operation`: `created`, `updated`, or `deleted`.
|
||||||
|
- `tenant_id`: tenant scope when the change is tenant-owned.
|
||||||
|
- `payload`: small, non-secret routing metadata that helps determine whether a
|
||||||
|
tombstone belongs to the requested view.
|
||||||
|
|
||||||
|
Sequence retention is explicit. Cleanup jobs must call
|
||||||
|
`prune_sequence_entries(...)` rather than deleting `core_change_sequence` rows
|
||||||
|
directly. Pruning records a retention floor per module, collection, and tenant
|
||||||
|
scope. Endpoints compare incoming watermarks with that floor; a watermark older
|
||||||
|
than the floor is not safe for incremental replay, so the endpoint must return a
|
||||||
|
full snapshot with `full: true`. A first-use `seq:0` watermark remains valid
|
||||||
|
until such a floor exists, even if unrelated collections have advanced the
|
||||||
|
global sequence.
|
||||||
|
|
||||||
|
### Cursor/Keyset Pages
|
||||||
|
|
||||||
|
Offset pagination remains supported for compatibility and for first page loads,
|
||||||
|
but it is not safe as the merge anchor for row-level deltas on page 2 and later.
|
||||||
|
When a delta-capable table can be paged beyond the first page, the endpoint
|
||||||
|
should expose keyset cursors:
|
||||||
|
|
||||||
|
- Core provides `encode_keyset_cursor`, `decode_keyset_cursor`, and
|
||||||
|
`keyset_query_fingerprint` in `govoplan_core.core.pagination`.
|
||||||
|
- Cursors are opaque to clients and contain the endpoint scope, query
|
||||||
|
fingerprint, and last-row keyset values.
|
||||||
|
- The fingerprint must include every query input that changes membership or
|
||||||
|
order: scope, tenant, page size, sort column, sort direction, and filters.
|
||||||
|
- Reusing a cursor with different sort or filter parameters must fail with a
|
||||||
|
client error rather than returning a mismatched slice.
|
||||||
|
- Responses may still include `page`, `page_size`, `pages`, and `total` for
|
||||||
|
existing UI components, but `cursor` identifies the current slice and
|
||||||
|
`next_cursor` is the safe anchor for the next slice.
|
||||||
|
- The first visit to an arbitrary page can use offset compatibility. The
|
||||||
|
response should include the start cursor for that page so later reloads and
|
||||||
|
delta requests use keyset semantics.
|
||||||
|
|
||||||
|
Concrete consumers:
|
||||||
|
|
||||||
|
- `GET /api/v1/files/delta`: without `since`, returns the current files/folders
|
||||||
|
snapshot for the requested owner/campaign scope. With `since=seq:<number>`,
|
||||||
|
returns changed files, changed folders, and tombstones for resources that
|
||||||
|
left the current view.
|
||||||
|
- `GET /api/v1/campaigns/delta`: returns accessible campaign rows and campaign
|
||||||
|
tombstones when ownership, sharing, or soft deletion removes a campaign from
|
||||||
|
the current list.
|
||||||
|
- `GET /api/v1/campaigns/{campaign_id}/workspace/delta`: returns a workspace
|
||||||
|
snapshot first, then changed campaign/version metadata and optional summary
|
||||||
|
refreshes when version, job, issue, or delivery-attempt changes invalidate the
|
||||||
|
workspace view.
|
||||||
|
- `GET /api/v1/campaigns/{campaign_id}/jobs/delta`: returns a paginated job
|
||||||
|
table snapshot first, then stable row deltas for cursor-backed job pages.
|
||||||
|
The job list also supports offset compatibility for first visits to a page and
|
||||||
|
returns `cursor`/`next_cursor` for stable reloads. Filtered, created, deleted,
|
||||||
|
or stale-watermark requests fall back to a full page snapshot when pagination
|
||||||
|
membership can shift.
|
||||||
|
- `GET /api/v1/admin/users/delta`, `/groups/delta`, `/roles/delta`,
|
||||||
|
`/system/roles/delta`, `/system/accounts/delta`, and `/api-keys/delta`:
|
||||||
|
return access administration row deltas with tombstones where rows leave the
|
||||||
|
visible view.
|
||||||
|
- `GET /api/v1/admin/system/settings/delta`: returns section deltas for system
|
||||||
|
defaults, tenant capability flags, language packages, privacy retention
|
||||||
|
policy, maintenance mode, and raw settings.
|
||||||
|
- `GET /api/v1/admin/tenant/settings/delta`: returns tenant-local setting
|
||||||
|
sections and also reports language-section changes when system language
|
||||||
|
packages or enabled language codes change.
|
||||||
|
- `GET /api/v1/admin/configuration-changes/delta`: returns changed
|
||||||
|
configuration requests and history records.
|
||||||
|
- `GET /api/v1/admin/audit` and `/api/v1/admin/audit/delta`: return append-only
|
||||||
|
audit events using the same scope, sort, and filter query parameters. The list
|
||||||
|
supports offset compatibility plus `cursor`/`next_cursor` keyset paging; the
|
||||||
|
delta endpoint can replay changes against a cursor-backed slice.
|
||||||
|
- `GET /api/v1/mail/settings/delta`: returns mail profile row deltas plus the
|
||||||
|
current scoped mail profile policy when profile-policy dependencies changed.
|
||||||
|
The WebUI consumes this for system, tenant, user, group, and campaign mail
|
||||||
|
settings panels.
|
||||||
|
- `GET /api/v1/files/connectors/settings/delta`: returns file connector
|
||||||
|
profile, credential, connector-space, and scoped connector-policy deltas.
|
||||||
|
Credential changes also include referencing profiles because profile rows
|
||||||
|
display credential-derived state.
|
||||||
|
|
||||||
|
Open retrofit scope:
|
||||||
|
|
||||||
|
- Additional module-specific settings pages should expose section deltas as
|
||||||
|
their settings APIs stabilize. Remaining likely candidates are future
|
||||||
|
booking/resource configuration pages and settings pages introduced by new
|
||||||
|
modules.
|
||||||
|
- Remaining high-volume tables should adopt the cursor/keyset contract before
|
||||||
|
enabling arbitrary-page row deltas. Current rollout follow-ups:
|
||||||
|
`govoplan-files#22` for large file-space server windows,
|
||||||
|
`govoplan-mail#9` for provider-aware mailbox message cursors,
|
||||||
|
`govoplan-calendar#7` for event-window deltas,
|
||||||
|
`govoplan-tenancy#1` for tenant administration row deltas, and
|
||||||
|
`govoplan-admin#2` for governance/module-operation list deltas.
|
||||||
|
|
||||||
## Module Responsibilities
|
## Module Responsibilities
|
||||||
|
|
||||||
A module owns one bounded feature area. A module can include both backend and WebUI code in the same repository so feature behavior and frontend integration evolve together.
|
A module owns one bounded feature area. A module can include both backend and WebUI code in the same repository so feature behavior and frontend integration evolve together.
|
||||||
@@ -195,6 +399,18 @@ NavItem(
|
|||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Core validates manifest shape when the platform registry is built. The current
|
||||||
|
supported manifest contract version is `1`, and frontend asset manifests use
|
||||||
|
contract version `1`. Registry validation rejects unsupported contract versions,
|
||||||
|
invalid module ids, duplicate dependency declarations, self-dependencies,
|
||||||
|
mismatched migration/frontend metadata, invalid frontend package names, and
|
||||||
|
frontend/nav routes that do not declare usable paths and labels.
|
||||||
|
|
||||||
|
Backend route contributions are also validated before they are mounted. Startup
|
||||||
|
routers and live module activation fail fast if two routers register the same
|
||||||
|
HTTP method and path. That keeps OpenAPI output and FastAPI route order from
|
||||||
|
silently masking a module collision.
|
||||||
|
|
||||||
## Database And Migrations
|
## Database And Migrations
|
||||||
|
|
||||||
Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest.
|
Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest.
|
||||||
@@ -209,6 +425,10 @@ Rules:
|
|||||||
- Optional module migrations may create multiple Alembic heads. Verification
|
- Optional module migrations may create multiple Alembic heads. Verification
|
||||||
should compare the database heads to the configured script heads instead of
|
should compare the database heads to the configured script heads instead of
|
||||||
assuming one linear revision when multiple modules are enabled.
|
assuming one linear revision when multiple modules are enabled.
|
||||||
|
- Treat migrations as release artifacts. Unreleased migrations may be squashed
|
||||||
|
or rewritten before a stable release; released revision IDs are immutable
|
||||||
|
once an installation may have recorded them. Each stable release records its
|
||||||
|
public migration heads in `docs/migration-release-baselines.json`.
|
||||||
|
|
||||||
## Install, Uninstall, And Catalogs
|
## Install, Uninstall, And Catalogs
|
||||||
|
|
||||||
@@ -290,6 +510,31 @@ capabilities with `usePlatformUiCapabilities("admin.sections")`, filters them
|
|||||||
by `anyOf`/`allOf`, and renders them without importing the contributing module's
|
by `anyOf`/`allOf`, and renders them without importing the contributing module's
|
||||||
components directly.
|
components directly.
|
||||||
|
|
||||||
|
The configurable dashboard follows the same pattern. Core contributes only a
|
||||||
|
minimal `/dashboard` fallback when no `dashboard` WebUI module is active. The
|
||||||
|
`govoplan-dashboard` module owns the real `/dashboard` route and collects
|
||||||
|
widgets exposed through the `dashboard.widgets` capability:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const dashboardWidgets: DashboardWidgetsUiCapability = {
|
||||||
|
widgets: [
|
||||||
|
{
|
||||||
|
id: "ops.health",
|
||||||
|
title: "Operations health",
|
||||||
|
moduleId: "ops",
|
||||||
|
defaultSize: "wide",
|
||||||
|
anyOf: ["ops:operations:read"],
|
||||||
|
render: ({ settings, refreshKey }) => createElement(OpsHealthWidget, { settings, refreshKey })
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Dashboard widgets are module contributions, not cross-module imports. A widget
|
||||||
|
may render components from its own module and core components only. The
|
||||||
|
dashboard module is responsible for layout, visibility, refresh context, and
|
||||||
|
future server-side layout persistence.
|
||||||
|
|
||||||
## Icon Rules
|
## Icon Rules
|
||||||
|
|
||||||
Icons are resolved centrally by core.
|
Icons are resolved centrally by core.
|
||||||
@@ -354,14 +599,221 @@ The repository includes `scripts/check_dependency_boundaries.py`. It enforces th
|
|||||||
- access source may not import files/mail/campaign internals
|
- access source may not import files/mail/campaign internals
|
||||||
- feature modules may not import access implementation internals
|
- feature modules may not import access implementation internals
|
||||||
- feature modules may not add new direct imports of sibling feature modules
|
- feature modules may not add new direct imports of sibling feature modules
|
||||||
- FastAPI routers may import the published
|
- FastAPI routers may import the published `govoplan_access.auth` dependency API
|
||||||
`govoplan_access.backend.auth.dependencies` dependency API
|
|
||||||
- the transitional allowlist is expected to stay empty
|
- the transitional allowlist is expected to stay empty
|
||||||
|
|
||||||
Any future exception is extraction debt and must be temporary, documented in the
|
Any future exception is extraction debt and must be temporary, documented in the
|
||||||
script with a reason, and removed when a capability/API/event contract replaces
|
script with a reason, and removed when a capability/API/event contract replaces
|
||||||
it.
|
it.
|
||||||
|
|
||||||
|
## Boundary Decision Register
|
||||||
|
|
||||||
|
These durable decisions close older exploratory core issues. Implementation
|
||||||
|
work should live in the owning module repositories once a boundary is clear.
|
||||||
|
|
||||||
|
Decision principles:
|
||||||
|
|
||||||
|
- Prefer connector-first when an external specialist system is likely to remain
|
||||||
|
the system of record.
|
||||||
|
- Create a native module only when GovOPlaN must own domain semantics,
|
||||||
|
permissions, audit, retention, configuration-package fragments, or workflow
|
||||||
|
state.
|
||||||
|
- Keep optional behavior behind core-mediated capabilities, events, DTOs, route
|
||||||
|
contributions, and UI contribution points.
|
||||||
|
- Do not create repositories just because a possible product area exists.
|
||||||
|
|
||||||
|
### Templates And Reporting
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#190`, `govoplan-templates#1`,
|
||||||
|
`govoplan-reporting#1`.
|
||||||
|
|
||||||
|
Decision: templates and reporting are separate modules.
|
||||||
|
|
||||||
|
`govoplan-templates` owns:
|
||||||
|
|
||||||
|
- reusable renderable templates for letters, permits, emails, forms, reports,
|
||||||
|
certificates, and notices
|
||||||
|
- template versioning, merge-field declarations, rendering profiles, output
|
||||||
|
format choices, and preview contracts
|
||||||
|
- template package fragments that other modules can reference
|
||||||
|
|
||||||
|
`govoplan-reporting` owns:
|
||||||
|
|
||||||
|
- report definitions, data selection, dashboards, BI views, scheduled outputs,
|
||||||
|
and export targets
|
||||||
|
- report permissions, report execution history, generated report evidence, and
|
||||||
|
report-specific retention inputs
|
||||||
|
- downstream export handoff to files, dataflow, connectors, or publication
|
||||||
|
surfaces
|
||||||
|
|
||||||
|
Boundary:
|
||||||
|
|
||||||
|
- Templates do not own data selection, aggregation, scheduling, or BI semantics.
|
||||||
|
- Reporting may call template rendering through a capability when a formatted
|
||||||
|
report output is needed.
|
||||||
|
- Campaign, mail, files, workflow, and cases use templates/reporting through
|
||||||
|
capabilities and DTOs, never direct imports.
|
||||||
|
|
||||||
|
### Sources, RSS, Datasources, And Dataflow
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#192`, `govoplan-core#197`,
|
||||||
|
`govoplan-core#198`, `govoplan-connectors#3`,
|
||||||
|
`govoplan-connectors#4`.
|
||||||
|
|
||||||
|
Decision: do not create `govoplan-datasources` or `govoplan-dataflow` until a
|
||||||
|
first executable use case proves that connector/reporting/workflow ownership is
|
||||||
|
too narrow.
|
||||||
|
|
||||||
|
First slice:
|
||||||
|
|
||||||
|
- `govoplan-connectors` owns RSS/Atom consume/emit connector profiles,
|
||||||
|
connector health, external references, source lifecycle metadata, and
|
||||||
|
source/publish capability boundaries.
|
||||||
|
- `govoplan-files` owns file-backed governed locations and uploaded/stored file
|
||||||
|
evidence.
|
||||||
|
- `govoplan-reporting` owns report/data views and scheduled outputs.
|
||||||
|
- `govoplan-workflow` owns process state, approvals, scheduling of process
|
||||||
|
steps, and human review.
|
||||||
|
|
||||||
|
Future `govoplan-datasources` is justified when GovOPlaN needs a broad source
|
||||||
|
catalogue for SQL databases, CSV/Excel files, APIs, RSS feeds, uploaded files,
|
||||||
|
and governed file locations with shared ownership, credentials, schema
|
||||||
|
discovery, refresh cadence, provenance, and permission boundaries.
|
||||||
|
|
||||||
|
Future `govoplan-dataflow` is justified when GovOPlaN needs first-class
|
||||||
|
pipelines for ingestion, transformation, validation, scheduling, lineage,
|
||||||
|
publication, audit events, reruns, and source-to-source workflows.
|
||||||
|
|
||||||
|
Monthly extraction/transformation work should start as a configuration package
|
||||||
|
and module collaboration across connectors, files, workflow, reporting, and
|
||||||
|
possibly templates. Create datasources/dataflow repositories only after that
|
||||||
|
package exposes repeated contracts that do not belong to an existing module.
|
||||||
|
|
||||||
|
### Calendar, Scheduling, And Appointments
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#193`, `govoplan-calendar#1`,
|
||||||
|
`govoplan-calendar#2`, `govoplan-scheduling#1`,
|
||||||
|
`govoplan-appointments#1`.
|
||||||
|
|
||||||
|
Decision: use three separate modules.
|
||||||
|
|
||||||
|
`govoplan-calendar` owns:
|
||||||
|
|
||||||
|
- calendar collections, events, recurrence, availability/free-busy, resources,
|
||||||
|
iCalendar import/export, CalDAV/Open-Xchange-style calendar adapters, and
|
||||||
|
calendar WebUI surfaces
|
||||||
|
|
||||||
|
`govoplan-scheduling` owns:
|
||||||
|
|
||||||
|
- Terminfindung, meeting-time polls, participant availability collection,
|
||||||
|
candidate-slot ranking, conflict explanations, reminders, and the handoff
|
||||||
|
from a selected slot to calendar/appointment/workflow modules
|
||||||
|
|
||||||
|
`govoplan-appointments` owns:
|
||||||
|
|
||||||
|
- Terminbuchung/fixed-slot appointment booking, appointment types, booking
|
||||||
|
rules, capacity, cancellation/no-show state, public/internal booking flows,
|
||||||
|
and appointment evidence
|
||||||
|
|
||||||
|
Boundary:
|
||||||
|
|
||||||
|
- Calendar provides time primitives and external calendar integration.
|
||||||
|
- Scheduling chooses a suitable time.
|
||||||
|
- Appointments owns booked appointment workflows and public/internal booking
|
||||||
|
semantics.
|
||||||
|
- Mail and notifications deliver invitations/reminders through capabilities.
|
||||||
|
|
||||||
|
### Forms And Workflow Handoff
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#194`, `govoplan-forms#1`.
|
||||||
|
|
||||||
|
Decision: forms are a reusable module boundary, with runtime behavior separated
|
||||||
|
from workflow semantics.
|
||||||
|
|
||||||
|
`govoplan-forms` owns:
|
||||||
|
|
||||||
|
- form definitions, schemas, validation rules, field visibility rules,
|
||||||
|
localization, versioning, admin editing, and reusable form package fragments
|
||||||
|
|
||||||
|
`govoplan-forms-runtime` owns, when implemented:
|
||||||
|
|
||||||
|
- public/internal submissions, drafts, submitted values, validation evidence,
|
||||||
|
attachment references, submission receipts, and handoff events
|
||||||
|
|
||||||
|
Boundary:
|
||||||
|
|
||||||
|
- Forms do not own cases, workflow transitions, tasks, or portal identity.
|
||||||
|
- Workflow/cases consume form submission events and evidence references.
|
||||||
|
- Files owns uploaded file storage and file permissions.
|
||||||
|
- Reporting/dataflow may consume submitted data through governed DTOs or
|
||||||
|
source lifecycle contracts.
|
||||||
|
|
||||||
|
### OpenDesk Integration Profile
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#195`, `govoplan-connectors#5`,
|
||||||
|
`govoplan-idm#1`, `govoplan-mail#5`, `govoplan-calendar#2`,
|
||||||
|
`govoplan-connectors#1`.
|
||||||
|
|
||||||
|
Decision: OpenDesk is an integration profile, not a monolithic module.
|
||||||
|
|
||||||
|
Ownership:
|
||||||
|
|
||||||
|
- identity: `govoplan-idm` plus `govoplan-access`
|
||||||
|
- mail/groupware: `govoplan-mail`
|
||||||
|
- calendar: `govoplan-calendar`
|
||||||
|
- files/documents: `govoplan-files` and later `govoplan-dms`
|
||||||
|
- projects/tasks: `govoplan-connectors` OpenProject connector first
|
||||||
|
- inventory/health/profile diagnostics: `govoplan-connectors`
|
||||||
|
|
||||||
|
The OpenDesk profile should describe required connector profiles, shared
|
||||||
|
identity assumptions, health checks, and optional module combinations. It must
|
||||||
|
not create direct module-to-module imports.
|
||||||
|
|
||||||
|
### Project Management And OpenProject
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#196`, `govoplan-connectors#1`.
|
||||||
|
|
||||||
|
Decision: connector-first. Do not create a native `govoplan-projects` module
|
||||||
|
yet.
|
||||||
|
|
||||||
|
OpenProject integration belongs in `govoplan-connectors` first:
|
||||||
|
|
||||||
|
- profile test
|
||||||
|
- project and work-package lookup
|
||||||
|
- external-reference storage
|
||||||
|
- selected publish/synchronize capabilities for tasks, workflow, or cases
|
||||||
|
|
||||||
|
A native project module is justified only if GovOPlaN needs to own project
|
||||||
|
semantics beyond cases, tasks, workflow, appointments, documents, and reporting,
|
||||||
|
for example portfolios, project budgets, project-level resource planning, or
|
||||||
|
governed project records that cannot remain in OpenProject.
|
||||||
|
|
||||||
|
### Public-Sector Integration Landscape
|
||||||
|
|
||||||
|
Tracking: `govoplan-core#186`, `govoplan-core#215`,
|
||||||
|
`govoplan-connectors#2`, `govoplan-connectors#3`.
|
||||||
|
|
||||||
|
Decision: core owns strategy and routing; connectors owns executable
|
||||||
|
integration catalogue entries and operator inventory.
|
||||||
|
|
||||||
|
Core documents:
|
||||||
|
|
||||||
|
- product-level integration strategy
|
||||||
|
- native-vs-connector decisions
|
||||||
|
- owning module routing
|
||||||
|
- roadmap sequencing
|
||||||
|
|
||||||
|
`govoplan-connectors` owns:
|
||||||
|
|
||||||
|
- connector entry schema
|
||||||
|
- external system catalogue
|
||||||
|
- connector profiles and diagnostics
|
||||||
|
- source consume/publish lifecycle
|
||||||
|
- external references
|
||||||
|
|
||||||
|
When a target needs executable behavior, create the implementation issue in the
|
||||||
|
owning module repository and keep only cross-module decisions in core.
|
||||||
|
|
||||||
## Module Lifecycle
|
## Module Lifecycle
|
||||||
|
|
||||||
Core exposes the installed module catalog through the admin API and WebUI. The
|
Core exposes the installed module catalog through the admin API and WebUI. The
|
||||||
@@ -565,6 +1017,8 @@ directory with:
|
|||||||
|
|
||||||
- `GOVOPLAN_INSTALLER_RUN_DIR`
|
- `GOVOPLAN_INSTALLER_RUN_DIR`
|
||||||
- `GOVOPLAN_DATABASE_URL`
|
- `GOVOPLAN_DATABASE_URL`
|
||||||
|
- `GOVOPLAN_DATABASE_URL_PGTOOLS` for PostgreSQL URLs converted to the
|
||||||
|
`postgresql://` form expected by `pg_dump`, `pg_restore`, and `psql`
|
||||||
- `GOVOPLAN_DATABASE_BACKUP_PATH`
|
- `GOVOPLAN_DATABASE_BACKUP_PATH`
|
||||||
- `GOVOPLAN_DATABASE_BACKUP_METADATA`
|
- `GOVOPLAN_DATABASE_BACKUP_METADATA`
|
||||||
|
|
||||||
@@ -631,10 +1085,20 @@ Backend verification from core:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign
|
||||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`scripts/check-focused.sh` runs npm with an isolated temporary npm user config
|
||||||
|
so developer-local npm settings do not create release-check warning noise.
|
||||||
|
|
||||||
|
Focused module contract and permutation verification:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-module-matrix.sh
|
||||||
|
```
|
||||||
|
|
||||||
Core WebUI host verification:
|
Core WebUI host verification:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
105
docs/POLICY_CONTRACTS.md
Normal file
105
docs/POLICY_CONTRACTS.md
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
# GovOPlaN Policy Contracts
|
||||||
|
|
||||||
|
GovOPlaN has several policy families that are moving out of core into owning
|
||||||
|
modules. The shared kernel contract keeps their decision and provenance shape
|
||||||
|
consistent while each module still owns its domain rules.
|
||||||
|
|
||||||
|
## Current Policy Inventory
|
||||||
|
|
||||||
|
| Policy area | Current owner | Runtime surface | Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Privacy retention | `govoplan-policy` routes with compatibility helpers in core | `/api/v1/admin/privacy-retention/policies/{scope}` and `/explain` | System, tenant, user, group, and campaign sources merge into the effective retention policy. Parent locks block lower-level widening. |
|
||||||
|
| Mail profile policy | `govoplan-mail` | `/api/v1/mail/policies/{scope}` | Uses the same source-step path format for system, tenant, owner, and campaign provenance. |
|
||||||
|
| RBAC/access policy | `govoplan-access` | access capabilities in `govoplan_core.core.access` | Permission decisions should use access capability contracts. Explain responses should adopt `PolicyDecision` when an API-level explanation is added. |
|
||||||
|
| Governance defaults | `govoplan-admin` plus `govoplan-access` materializer | admin settings, governance template routes, access materialization capability | System governance can block tenant-local groups, roles, and API keys. |
|
||||||
|
| Delegation and ownership policy | access/campaign/mail/files modules | capability checks and owner-scoped APIs | Source provenance should use this contract when policies become externally explainable. |
|
||||||
|
|
||||||
|
## Policy Decision
|
||||||
|
|
||||||
|
The shared DTO lives in `govoplan_core.core.policy.PolicyDecision`.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"allowed": false,
|
||||||
|
"reason": "Parent retention policy locks lower-level changes.",
|
||||||
|
"source_path": [
|
||||||
|
{
|
||||||
|
"scope_type": "system",
|
||||||
|
"scope_id": null,
|
||||||
|
"path": "system",
|
||||||
|
"label": "System",
|
||||||
|
"applied_fields": ["allow_lower_level_limits"],
|
||||||
|
"policy": {}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"requirements": ["raw_campaign_json_retention_days"],
|
||||||
|
"details": {
|
||||||
|
"blocked_fields": ["raw_campaign_json_retention_days"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`allowed` is the effective answer for the checked action. `reason` is a stable,
|
||||||
|
human-readable summary. `source_path` lists the policy sources that explain the
|
||||||
|
answer. `requirements` lists machine-readable blockers or prerequisites, and
|
||||||
|
`details` carries domain-specific structured context.
|
||||||
|
|
||||||
|
Every source step should be concrete enough for an operator to understand the
|
||||||
|
decision without knowing internal merge rules. Use real scope labels such as
|
||||||
|
`System`, `Tenant`, `Owner user`, `Group`, or a campaign/profile name. Include
|
||||||
|
the stable `path`, the fields applied by that step, and the local policy
|
||||||
|
fragment that caused them. This lets UIs render explanations like
|
||||||
|
`System: Allow > Tenant: Deny without override` without additional lookups.
|
||||||
|
If a policy family cannot expose the full local fragment for security reasons,
|
||||||
|
it must still include a redacted structured value that identifies the applied
|
||||||
|
field and the effective allow/deny or lock state.
|
||||||
|
|
||||||
|
## Source Path Format
|
||||||
|
|
||||||
|
Policy source paths are stable string identifiers for provenance steps:
|
||||||
|
|
||||||
|
- `system`
|
||||||
|
- `<scope_type>:<url-encoded-scope-id>`
|
||||||
|
|
||||||
|
Supported scope types are `system`, `tenant`, `user`, `group`, and `campaign`.
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- `tenant:4a45b4fe-1d86-43ce-9d10-6022333f4d4b`
|
||||||
|
- `campaign:campaign%2Fwith%20space`
|
||||||
|
|
||||||
|
Use `policy_source_path()` and `parse_policy_source_path()` instead of building
|
||||||
|
or splitting these strings manually.
|
||||||
|
|
||||||
|
## Retention Explain Endpoint
|
||||||
|
|
||||||
|
`GET /api/v1/admin/privacy-retention/policies/{scope_type}/explain` returns:
|
||||||
|
|
||||||
|
- `scope_type` and optional `scope_id`
|
||||||
|
- `decision`, using the shared `PolicyDecision` shape
|
||||||
|
- `effective_policy`
|
||||||
|
- optional `parent_policy`
|
||||||
|
- `effective_policy_sources`
|
||||||
|
- `parent_policy_sources`
|
||||||
|
- `blocked_fields`
|
||||||
|
|
||||||
|
The endpoint is read-only. Enforcement remains in the existing policy write
|
||||||
|
path. For lower-level scopes, `blocked_fields` is derived from the parent
|
||||||
|
policy's `allow_lower_level_limits`; clients can use it to disable local
|
||||||
|
controls before attempting a write.
|
||||||
|
|
||||||
|
## Frontend Contract
|
||||||
|
|
||||||
|
Policy UIs must:
|
||||||
|
|
||||||
|
- render effective source provenance when `effective_policy_sources` is present
|
||||||
|
- display a field-level path when the source data is shown next to a specific
|
||||||
|
setting, using concrete source labels and stop at the first non-overridable
|
||||||
|
deny/lock
|
||||||
|
- disable local field controls when the parent policy sets that field's
|
||||||
|
lower-level limit to `false`
|
||||||
|
- avoid sending locked fields or re-enable attempts in save payloads
|
||||||
|
- show inherited values separately from local overrides
|
||||||
|
|
||||||
|
The core WebUI helper `privacyRetentionParentAllowsField()` centralizes the
|
||||||
|
field-lock decision used by the retention editor and its lightweight module
|
||||||
|
tests.
|
||||||
134
docs/POSTBOX_E2EE_ARCHITECTURE.md
Normal file
134
docs/POSTBOX_E2EE_ARCHITECTURE.md
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
# Postbox End-To-End Encryption Architecture
|
||||||
|
|
||||||
|
This document records the strategic encryption target for GovOPlaN postboxes.
|
||||||
|
It does not require the first postbox implementation to ship full E2EE, but it
|
||||||
|
defines the architecture so early data models and APIs do not make the stronger
|
||||||
|
model impossible.
|
||||||
|
|
||||||
|
The core principle is that a postbox can become a trusted administrative
|
||||||
|
communication channel without requiring the server to see plaintext content.
|
||||||
|
The server may route, store, authorize, audit, retain, and expire messages while
|
||||||
|
message bodies and attachments remain client-encrypted.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- asynchronous encrypted delivery for internal and portal-facing postboxes
|
||||||
|
- personal, organizational, role-bound, and function-bound postboxes
|
||||||
|
- attachments encrypted with the message
|
||||||
|
- access based on current role/function membership when configured
|
||||||
|
- honest retraction and expiry semantics
|
||||||
|
- auditable key access, delivery, and fetch events
|
||||||
|
- support for external recipients without platform accounts
|
||||||
|
- replaceable identity and trust providers
|
||||||
|
|
||||||
|
## Envelope Model
|
||||||
|
|
||||||
|
The target model is envelope encryption:
|
||||||
|
|
||||||
|
- Generate one random data encryption key per message or attachment set.
|
||||||
|
- Encrypt content with an authenticated encryption algorithm.
|
||||||
|
- Wrap the data encryption key for each authorized recipient or role mailbox.
|
||||||
|
- Store only ciphertext, wrapped keys, signed manifests, and governed metadata
|
||||||
|
on the server.
|
||||||
|
|
||||||
|
Algorithm choices should remain replaceable behind a crypto profile. The first
|
||||||
|
profile should prefer standard, reviewed primitives such as HPKE for key
|
||||||
|
wrapping and AEAD encryption for content.
|
||||||
|
|
||||||
|
## Identity And Device Keys
|
||||||
|
|
||||||
|
The platform should distinguish:
|
||||||
|
|
||||||
|
- account identity
|
||||||
|
- tenant membership
|
||||||
|
- role/function assignment
|
||||||
|
- device key
|
||||||
|
- postbox binding
|
||||||
|
|
||||||
|
Identity providers and directories can authenticate users and provide membership
|
||||||
|
facts, but they must not see postbox private keys or message plaintext.
|
||||||
|
|
||||||
|
The trust layer should provide:
|
||||||
|
|
||||||
|
- public key directory
|
||||||
|
- account or identity signing keys
|
||||||
|
- per-device encryption keys
|
||||||
|
- device registration and revocation
|
||||||
|
- key rotation and epoch tracking
|
||||||
|
- recovery policy hooks
|
||||||
|
|
||||||
|
## Role And Function Postboxes
|
||||||
|
|
||||||
|
Role-bound access needs special handling. A postbox can be bound to an
|
||||||
|
organizational unit and a role or function. Current members can access current
|
||||||
|
messages according to policy; former members should lose access to not-yet
|
||||||
|
fetched material when revocation is still technically enforceable.
|
||||||
|
|
||||||
|
The target design should support role encryption keys or an equivalent
|
||||||
|
rewrapping service:
|
||||||
|
|
||||||
|
- sender encrypts the content key for the role/function postbox
|
||||||
|
- access service verifies current membership and required assurance
|
||||||
|
- trust service rewraps the content key to the actor's current device key
|
||||||
|
- audit records the key access decision and fetch event
|
||||||
|
|
||||||
|
Key epochs are required when role membership changes. Older messages may remain
|
||||||
|
readable according to policy, but new access must use the current epoch.
|
||||||
|
|
||||||
|
## External Recipients
|
||||||
|
|
||||||
|
External recipients may need one-time or time-limited access without a full
|
||||||
|
platform account. The target model should support capability links or invitation
|
||||||
|
tokens that are:
|
||||||
|
|
||||||
|
- scoped to specific message or attachment resources
|
||||||
|
- time-limited
|
||||||
|
- optionally one-time
|
||||||
|
- protected by an out-of-band secret, passphrase, or stronger external identity
|
||||||
|
proof
|
||||||
|
- revocable before key fetch
|
||||||
|
- fully audited
|
||||||
|
|
||||||
|
## Retraction Semantics
|
||||||
|
|
||||||
|
GovOPlaN should be honest about retraction.
|
||||||
|
|
||||||
|
Before a recipient fetches a key or decrypts content, the system can revoke
|
||||||
|
tokens, remove wrapped-key access, expire links, and delete ciphertext according
|
||||||
|
to retention policy.
|
||||||
|
|
||||||
|
After a recipient has decrypted or copied plaintext, the system cannot make the
|
||||||
|
recipient forget it. The platform can only record access, revoke future access,
|
||||||
|
notify parties, and apply legal or organizational controls.
|
||||||
|
|
||||||
|
The UI must explain this distinction whenever it offers expiry, retraction, or
|
||||||
|
message withdrawal.
|
||||||
|
|
||||||
|
## Server Responsibilities
|
||||||
|
|
||||||
|
The server remains important even when content is encrypted:
|
||||||
|
|
||||||
|
- store ciphertext and signed manifests
|
||||||
|
- store routing and policy metadata
|
||||||
|
- enforce access before key release or rewrapping
|
||||||
|
- provide public key directory access
|
||||||
|
- emit notifications without plaintext content
|
||||||
|
- record audit events
|
||||||
|
- enforce retention and expiry where possible
|
||||||
|
- expose diagnostics for delivery and key-access failures
|
||||||
|
|
||||||
|
## Module Ownership
|
||||||
|
|
||||||
|
- `govoplan-postbox` owns postbox bindings, postbox messages, message metadata,
|
||||||
|
and postbox UI.
|
||||||
|
- `govoplan-identity-trust` owns device keys, public key directory, key epochs,
|
||||||
|
and assurance integration.
|
||||||
|
- `govoplan-access` owns current identity, membership, function, delegation, and
|
||||||
|
permission decisions.
|
||||||
|
- `govoplan-policy` owns retention, retraction, and security policy decisions.
|
||||||
|
- `govoplan-audit` owns durable audit traces.
|
||||||
|
- `govoplan-files` owns managed file storage when encrypted postbox attachments
|
||||||
|
are backed by file objects.
|
||||||
|
|
||||||
|
No module should import another module's internals to decrypt content. All
|
||||||
|
interaction must use capabilities, DTOs, and audited service contracts.
|
||||||
276
docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md
Normal file
276
docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
# Public-Sector Integration Strategy
|
||||||
|
|
||||||
|
GovOPlaN should integrate with the existing public-sector software landscape
|
||||||
|
before deciding to replace specialist workflows. This document is the core
|
||||||
|
strategy index. The executable connector catalogue lives in
|
||||||
|
`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`.
|
||||||
|
|
||||||
|
## Strategy Labels
|
||||||
|
|
||||||
|
Use one or more of these labels for every external system family:
|
||||||
|
|
||||||
|
- `integrate`: GovOPlaN talks to the system through a stable API/protocol.
|
||||||
|
- `link`: GovOPlaN stores external references and opens the external system for
|
||||||
|
source-of-truth work.
|
||||||
|
- `import`: GovOPlaN consumes data or files into governed module storage.
|
||||||
|
- `synchronize`: GovOPlaN keeps selected records aligned both ways or through a
|
||||||
|
source-of-truth rule.
|
||||||
|
- `replace selected workflow`: GovOPlaN may own a narrow workflow where the
|
||||||
|
external product is weak, but does not replace the whole product family.
|
||||||
|
- `no first-class support`: GovOPlaN only stores manual references unless a
|
||||||
|
deployment project creates a specific connector.
|
||||||
|
|
||||||
|
## Initial Classification
|
||||||
|
|
||||||
|
| System family | Examples | Default strategy | Likely owner |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| File providers | SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3 | integrate, import, link | `govoplan-files`, connector inventory in `govoplan-connectors` |
|
||||||
|
| Project/task management | OpenProject, Jira, Redmine, Microsoft Planner | link, synchronize selected records, replace selected workflow only after proof | `govoplan-connectors`, later `govoplan-tasks` or workflow modules |
|
||||||
|
| Identity providers | LDAP, Active Directory, OIDC, SAML, OpenDesk IDM | integrate, synchronize | `govoplan-idm`, `govoplan-access` |
|
||||||
|
| Mail and groupware | IMAP/SMTP, Open-Xchange, Exchange/M365, CalDAV/CardDAV | integrate, link | `govoplan-mail`, `govoplan-calendar`, `govoplan-connectors` |
|
||||||
|
| DMS/e-file/archive | d.velop/d.3, enaio, ELO, Fabasoft, CMIS, VIS/eAkte | link, import, synchronize selected metadata | `govoplan-dms`, `govoplan-files`, `govoplan-records` |
|
||||||
|
| ERP/finance/procurement | SAP, MACH, Infoma, DATEV, procurement feeds | export, import, synchronize; do not replace by default | `govoplan-erp`, `govoplan-procurement`, `govoplan-ledger`, `govoplan-payments` |
|
||||||
|
| Public-sector transport | FIT-Connect, XTA/OSCI, Peppol access points | integrate, publish, receive | dedicated protocol modules plus `govoplan-connectors` inventory |
|
||||||
|
| Standards registries | XRepository, XOE/V catalogues | link, import metadata/cache | `govoplan-connectors`, `govoplan-xoev` |
|
||||||
|
| Publication/data exchange | RSS, open-data APIs, API feeds, CSV/Excel drops | consume, publish, transform | proposed `govoplan-datasources`, proposed `govoplan-dataflow`, `govoplan-reporting` |
|
||||||
|
| Collaboration suites | Matrix, Jitsi, BigBlueButton, Nextcloud Talk, Collabora/OnlyOffice | integrate, link; native behavior only for governed evidence | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow` |
|
||||||
|
| Specialist Fachverfahren | register-specific and domain-specific systems | link first; integrate/import when a real project supplies contracts | domain module or deployment-specific connector |
|
||||||
|
|
||||||
|
## Landscape Catalogue
|
||||||
|
|
||||||
|
This catalogue is intentionally implementation-oriented. Each entry records the
|
||||||
|
first API/auth/data assumptions needed to turn an inventory entry into a
|
||||||
|
connector or module issue.
|
||||||
|
|
||||||
|
### Citizen And Service Portals
|
||||||
|
|
||||||
|
- Strategy: integrate/link first; replace selected intake workflow only when a
|
||||||
|
GovOPlaN portal package owns the complete journey.
|
||||||
|
- Protocol/API surface: REST/JSON APIs, form submission webhooks, OIDC/SAML
|
||||||
|
login, eID interfaces where available, file-upload callbacks, case-status
|
||||||
|
callbacks.
|
||||||
|
- Auth model: OIDC/SAML service clients, signed webhook secrets, tenant-scoped
|
||||||
|
API keys, later eID/trust-provider handoff.
|
||||||
|
- Data shape: applicant identity reference, application form payload,
|
||||||
|
attachment references, consent declarations, status events, receipt IDs.
|
||||||
|
- Deployment assumptions: externally reachable HTTPS, reverse proxy, portal DMZ
|
||||||
|
separation, strict CSRF/origin settings, large upload path.
|
||||||
|
- Risks: personal data exposure, duplicate identity mapping, partial
|
||||||
|
submissions, upload malware, inconsistent portal status models.
|
||||||
|
- MVP test path: submit a test application with one file, create a form
|
||||||
|
submission/case/task stub, return a receipt and status reference.
|
||||||
|
- Owner/priority: `govoplan-portal`, `govoplan-forms-runtime`,
|
||||||
|
`govoplan-files`, Wave 1.
|
||||||
|
|
||||||
|
### DMS, E-File, Records, And Archives
|
||||||
|
|
||||||
|
- Strategy: link/import/synchronize selected metadata; do not replace the DMS by
|
||||||
|
default.
|
||||||
|
- Protocol/API surface: CMIS, WebDAV, vendor REST APIs, S3/object archive
|
||||||
|
staging, file-plan export/import, archive handoff APIs.
|
||||||
|
- Auth model: service accounts, OAuth/OIDC where supported, mTLS for regulated
|
||||||
|
archives, secret references for vendor tokens.
|
||||||
|
- Data shape: document ID, version, file-plan/classification code, retention
|
||||||
|
metadata, owner/case reference, external URL, checksum, lock/legal-hold state.
|
||||||
|
- Deployment assumptions: usually internal network or VPN, strict storage
|
||||||
|
quotas, existing retention policies, archive immutability requirements.
|
||||||
|
- Risks: record duplication, broken legal hold, permission mismatch, version
|
||||||
|
drift, destructive retention/export mistakes.
|
||||||
|
- MVP test path: create a connector inventory entry, test read-only metadata
|
||||||
|
lookup, link one GovOPlaN file/case evidence item to an external document.
|
||||||
|
- Owner/priority: `govoplan-dms`, `govoplan-records`, `govoplan-files`,
|
||||||
|
`govoplan-connectors`, Wave 2/5.
|
||||||
|
|
||||||
|
### ERP, Finance, Procurement, And Accounting
|
||||||
|
|
||||||
|
- Strategy: export/import/synchronize selected records; replacement only by
|
||||||
|
narrow domain decision.
|
||||||
|
- Protocol/API surface: vendor REST/SOAP APIs, CSV/XML batch exchange, SFTP,
|
||||||
|
XRechnung/Peppol, XBestellung/procurement feeds, payment reconciliation files.
|
||||||
|
- Auth model: service accounts, client certificates, mTLS, SFTP keys, token
|
||||||
|
references, environment-specific account separation.
|
||||||
|
- Data shape: debtor/creditor reference, payment request, invoice, order,
|
||||||
|
budget/cost-center code, booking status, receipt/evidence reference.
|
||||||
|
- Deployment assumptions: batch windows, finance-system approval workflows,
|
||||||
|
test tenants often separated from production by vendor process.
|
||||||
|
- Risks: financial posting errors, double export, tax/legal data retention,
|
||||||
|
inconsistent master data, irreversible accounting handoff.
|
||||||
|
- MVP test path: dry-run export of one payment/accounting handoff file with
|
||||||
|
checksum, validation report, and no remote posting.
|
||||||
|
- Owner/priority: `govoplan-payments`, `govoplan-ledger`,
|
||||||
|
`govoplan-xrechnung`, `govoplan-erp`, `govoplan-procurement`, Wave 1/6.
|
||||||
|
|
||||||
|
### Identity, IAM, And Directory Services
|
||||||
|
|
||||||
|
- Strategy: integrate/synchronize; access remains GovOPlaN's local
|
||||||
|
authorization boundary.
|
||||||
|
- Protocol/API surface: LDAP, Active Directory, SCIM, OIDC, SAML, OpenDesk IDM
|
||||||
|
APIs, group membership sync, account deactivation feeds.
|
||||||
|
- Auth model: bind accounts, service clients, OIDC/SAML metadata, SCIM tokens,
|
||||||
|
certificate-backed clients where required.
|
||||||
|
- Data shape: account, user, group, membership, role claim, tenant/org-unit
|
||||||
|
mapping, status, external directory ID.
|
||||||
|
- Deployment assumptions: directory is usually internal; identity provider may
|
||||||
|
be organization-wide and not GovOPlaN-owned.
|
||||||
|
- Risks: privilege escalation through group mapping, stale memberships, account
|
||||||
|
collision, deprovisioning latency, tenant-boundary mistakes.
|
||||||
|
- MVP test path: read-only directory profile test, map one external group to a
|
||||||
|
tenant group, show a dry-run membership diff.
|
||||||
|
- Owner/priority: `govoplan-idm`, `govoplan-access`, Wave 1.
|
||||||
|
|
||||||
|
### Groupware, Mail, Calendar, And Collaboration
|
||||||
|
|
||||||
|
- Strategy: integrate/link; native behavior only where GovOPlaN needs governed
|
||||||
|
evidence or process state.
|
||||||
|
- Protocol/API surface: IMAP/SMTP, CalDAV/CardDAV, Open-Xchange APIs, Microsoft
|
||||||
|
Graph/EWS, Matrix APIs, Jitsi/BigBlueButton APIs, Collabora/OnlyOffice
|
||||||
|
integration points.
|
||||||
|
- Auth model: service accounts, delegated OAuth/OIDC, app passwords, mailbox
|
||||||
|
credentials, groupware-specific tokens, secret references.
|
||||||
|
- Data shape: mailbox folder/message references, event/free-busy data, meeting
|
||||||
|
URL, chat room ID, participant list, document-editing session reference.
|
||||||
|
- Deployment assumptions: often internal/existing tenant infrastructure; mail
|
||||||
|
and calendar may be separate from identity even in OpenDesk-style stacks.
|
||||||
|
- Risks: mail credential exposure, calendar privacy, double invitations, room
|
||||||
|
booking conflicts, chat/document data escaping retention rules.
|
||||||
|
- MVP test path: profile test for mailbox/calendar reachability, read-only
|
||||||
|
folder/free-busy lookup, create a non-production event/message draft.
|
||||||
|
- Owner/priority: `govoplan-mail`, `govoplan-calendar`,
|
||||||
|
`govoplan-connectors`, Wave 1/2.
|
||||||
|
|
||||||
|
### Payment And Public Cashier Systems
|
||||||
|
|
||||||
|
- Strategy: integrate/export/import; keep the payment provider or cashier as
|
||||||
|
source of settlement truth.
|
||||||
|
- Protocol/API surface: payment provider APIs, redirect/callback flows,
|
||||||
|
reconciliation files, SEPA/export formats, cash-register/cashier interfaces.
|
||||||
|
- Auth model: provider API keys, signed webhooks, client certificates, mTLS,
|
||||||
|
tenant-specific merchant accounts.
|
||||||
|
- Data shape: payment intent, amount/currency, payer reference, provider
|
||||||
|
transaction ID, settlement status, receipt, refund/cancellation reference.
|
||||||
|
- Deployment assumptions: public callback URLs, strict environment separation,
|
||||||
|
PCI-sensitive providers, finance reconciliation cadence.
|
||||||
|
- Risks: duplicate charges, callback replay, amount mismatch, refund workflow
|
||||||
|
gaps, evidence-retention mistakes.
|
||||||
|
- MVP test path: sandbox payment intent, signed callback verification, receipt
|
||||||
|
evidence link, reconciliation dry-run.
|
||||||
|
- Owner/priority: `govoplan-payments`, `govoplan-ledger`, Wave 1/6.
|
||||||
|
|
||||||
|
### Reporting, BI, Open Data, And Publication
|
||||||
|
|
||||||
|
- Strategy: consume/publish/transform; native reporting owns GovOPlaN views, not
|
||||||
|
every external BI product.
|
||||||
|
- Protocol/API surface: SQL read replicas, CSV/Excel export/import, REST APIs,
|
||||||
|
RSS/Atom, open-data APIs, SFTP/WebDAV publication targets.
|
||||||
|
- Auth model: read-only DB users, API tokens, SFTP keys, OAuth clients, public
|
||||||
|
anonymous publication profiles where appropriate.
|
||||||
|
- Data shape: dataset metadata, schema/version, report parameters, generated
|
||||||
|
file references, publication URL, freshness/lineage, validation results.
|
||||||
|
- Deployment assumptions: publication can be public or internal; generated
|
||||||
|
datasets need retention and provenance.
|
||||||
|
- Risks: leaking restricted data, stale publications, schema drift, expensive
|
||||||
|
queries, untraceable manual transformations.
|
||||||
|
- MVP test path: publish one report/export as a governed file plus RSS/Atom
|
||||||
|
entry with checksum, timestamp, and permission check.
|
||||||
|
- Owner/priority: `govoplan-reporting`, `govoplan-connectors`, possible future
|
||||||
|
`govoplan-datasources`/`govoplan-dataflow`, Wave 2.
|
||||||
|
|
||||||
|
### Public-Sector Protocols And Registries
|
||||||
|
|
||||||
|
- Strategy: integrate/publish/receive; protocol modules own protocol semantics.
|
||||||
|
- Protocol/API surface: FIT-Connect, XTA/OSCI, XRepository, XOE/V, XRechnung,
|
||||||
|
XBestellung, Peppol, registry-specific Fachverfahren APIs.
|
||||||
|
- Auth model: certificates, mTLS, service accounts, destination credentials,
|
||||||
|
protocol-specific trust anchors and key rotation.
|
||||||
|
- Data shape: transport envelope, payload schema/version, destination IDs,
|
||||||
|
receipt/acknowledgement, message status, standard-specific metadata.
|
||||||
|
- Deployment assumptions: regulated trust chains, test/prod endpoint
|
||||||
|
separation, formal onboarding, strict logging and retention expectations.
|
||||||
|
- Risks: invalid schemas, failed delivery receipts, certificate expiry, wrong
|
||||||
|
destination routing, protocol version drift.
|
||||||
|
- MVP test path: validate a sample payload against a schema, test endpoint
|
||||||
|
reachability in sandbox, store receipt/evidence reference.
|
||||||
|
- Owner/priority: `govoplan-fit-connect`, `govoplan-xoev`,
|
||||||
|
`govoplan-xrechnung`, `govoplan-xta-osci`, `govoplan-connectors`, Wave 1/2.
|
||||||
|
|
||||||
|
### File Providers And Shared Storage
|
||||||
|
|
||||||
|
- Strategy: integrate/import/link; files module owns GovOPlaN file semantics.
|
||||||
|
- Protocol/API surface: SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3,
|
||||||
|
local/object storage profiles.
|
||||||
|
- Auth model: service accounts, user credentials, app tokens, OAuth where
|
||||||
|
supported, secret references, environment variables for deployment-managed
|
||||||
|
credentials.
|
||||||
|
- Data shape: file ID/path, provider object ID, checksum, MIME type, size,
|
||||||
|
version/ETag, owner, permission snapshot, imported file reference.
|
||||||
|
- Deployment assumptions: internal networks, large files, existing shares,
|
||||||
|
variable permissions, provider-specific rate limits.
|
||||||
|
- Risks: permission mismatch, stale imports, overwrites, duplicate files, path
|
||||||
|
traversal, storage growth.
|
||||||
|
- MVP test path: profile test, list folder, import one file into governed
|
||||||
|
storage, keep provider reference and checksum.
|
||||||
|
- Owner/priority: `govoplan-files`, `govoplan-connectors`, Wave 0/1.
|
||||||
|
|
||||||
|
### Project, Task, And Case-Adjacent Systems
|
||||||
|
|
||||||
|
- Strategy: connector-first for OpenProject/Jira/Redmine; native module only
|
||||||
|
when GovOPlaN owns project semantics.
|
||||||
|
- Protocol/API surface: OpenProject API v3, webhooks, Jira/Redmine REST APIs,
|
||||||
|
Microsoft Graph for Planner/Project where applicable.
|
||||||
|
- Auth model: API tokens, OAuth/OIDC apps, webhook secrets, service accounts.
|
||||||
|
- Data shape: project ID, work package/task ID, status, assignee reference,
|
||||||
|
external URL, version/lock token, publish/sync trace.
|
||||||
|
- Deployment assumptions: external project tool remains source of truth for
|
||||||
|
broad project management; GovOPlaN links selected records.
|
||||||
|
- Risks: task duplication, bidirectional sync conflicts, permission mismatch,
|
||||||
|
over-mirroring comments/attachments.
|
||||||
|
- MVP test path: OpenProject profile test, project/work-package lookup,
|
||||||
|
external-reference round-trip.
|
||||||
|
- Owner/priority: `govoplan-connectors`, later `govoplan-tasks`/workflow/cases
|
||||||
|
consumers, Wave 0/2.
|
||||||
|
|
||||||
|
### Specialist Fachverfahren
|
||||||
|
|
||||||
|
- Strategy: link first; integrate/import only when a deployment project supplies
|
||||||
|
concrete contracts and a domain owner.
|
||||||
|
- Protocol/API surface: vendor APIs, CSV/XML batch imports, SFTP, database
|
||||||
|
views, message queues, protocol-specific transports.
|
||||||
|
- Auth model: usually service accounts, VPN, mTLS, SFTP keys, or vendor tokens.
|
||||||
|
- Data shape: domain-specific record IDs, status, applicant/person references,
|
||||||
|
file/evidence references, case/status events.
|
||||||
|
- Deployment assumptions: strongly local/vendor-specific, often no stable test
|
||||||
|
API, data model differs by jurisdiction.
|
||||||
|
- Risks: brittle vendor contracts, legal source-of-truth ambiguity, high
|
||||||
|
customization cost, migration expectations.
|
||||||
|
- MVP test path: inventory entry and manual external-reference link; require a
|
||||||
|
project-specific connector issue before automation.
|
||||||
|
- Owner/priority: domain module or deployment-specific connector, case by case.
|
||||||
|
|
||||||
|
## Prioritization Rules
|
||||||
|
|
||||||
|
1. Start with connectors that unblock Wave 0 or Wave 1 reference journeys.
|
||||||
|
2. Prefer open standards and self-hosted/open-source APIs where they are common
|
||||||
|
in public-sector deployments.
|
||||||
|
3. Treat inventory-only entries as useful because operators need a map of their
|
||||||
|
software landscape even before automation exists.
|
||||||
|
4. Keep connector code in the owning connector/protocol module. Domain modules
|
||||||
|
consume capabilities, DTOs, external references, and events through core.
|
||||||
|
5. Every executable connector needs health diagnostics, secret-reference
|
||||||
|
handling, lifecycle state, audit events, and retirement behavior.
|
||||||
|
|
||||||
|
## Connector Catalogue Handoff
|
||||||
|
|
||||||
|
`govoplan-connectors` owns the detailed catalogue entry shape:
|
||||||
|
|
||||||
|
- connector type key
|
||||||
|
- category and owner module
|
||||||
|
- supported directions and trigger modes
|
||||||
|
- credential and secret handling
|
||||||
|
- health check and diagnostics payload
|
||||||
|
- external-reference shape
|
||||||
|
- required capabilities and optional module combinations
|
||||||
|
- lifecycle support
|
||||||
|
|
||||||
|
Core should only keep strategy, routing, and cross-module architecture notes.
|
||||||
|
Connector implementation and public-sector target inventory belong in
|
||||||
|
`govoplan-connectors`.
|
||||||
@@ -1,291 +0,0 @@
|
|||||||
# Multi Seal Mail - Current RBAC and Resource-Access Model
|
|
||||||
|
|
||||||
**Updated:** 2026-06-16
|
|
||||||
**Current migration head:** `f5a6b7c8d9e0`
|
|
||||||
|
|
||||||
## Authorization Equation
|
|
||||||
|
|
||||||
An operation is permitted only when every applicable layer allows it:
|
|
||||||
|
|
||||||
```text
|
|
||||||
effective role/API-key capability
|
|
||||||
AND resource ownership/share access
|
|
||||||
AND workflow state
|
|
||||||
AND active governance/policy constraints
|
|
||||||
```
|
|
||||||
|
|
||||||
RBAC answers what an actor may do. ACLs answer which resource the actor may do it to. Workflow state and policy decide whether the operation is currently valid.
|
|
||||||
|
|
||||||
## Identity and Scope
|
|
||||||
|
|
||||||
```text
|
|
||||||
Account global login identity
|
|
||||||
+- User membership tenant-local identity
|
|
||||||
+- direct tenant roles
|
|
||||||
+- active group memberships
|
|
||||||
| +- inherited tenant roles
|
|
||||||
+- tenant-local API keys
|
|
||||||
|
|
||||||
Account
|
|
||||||
+- direct system-role assignments
|
|
||||||
```
|
|
||||||
|
|
||||||
A browser session has one active tenant membership. System privileges do not silently grant tenant data access. API keys remain tenant-local and receive the intersection of their configured scopes and their owner's live tenant scopes on every request.
|
|
||||||
|
|
||||||
## Wildcards
|
|
||||||
|
|
||||||
```text
|
|
||||||
tenant:* every canonical tenant permission
|
|
||||||
system:* every canonical system permission
|
|
||||||
* legacy alias interpreted as tenant:* only
|
|
||||||
```
|
|
||||||
|
|
||||||
Tenant wildcards never grant system permissions.
|
|
||||||
|
|
||||||
## Canonical Tenant Permissions - 53
|
|
||||||
|
|
||||||
### Campaigns
|
|
||||||
|
|
||||||
```text
|
|
||||||
campaign:read
|
|
||||||
campaign:create
|
|
||||||
campaign:update
|
|
||||||
campaign:copy
|
|
||||||
campaign:archive
|
|
||||||
campaign:delete
|
|
||||||
campaign:share
|
|
||||||
campaign:validate
|
|
||||||
campaign:build
|
|
||||||
campaign:review
|
|
||||||
campaign:send_test
|
|
||||||
campaign:queue
|
|
||||||
campaign:control
|
|
||||||
campaign:send
|
|
||||||
campaign:retry
|
|
||||||
campaign:reconcile
|
|
||||||
```
|
|
||||||
|
|
||||||
### Recipients
|
|
||||||
|
|
||||||
```text
|
|
||||||
recipients:read
|
|
||||||
recipients:write
|
|
||||||
recipients:import
|
|
||||||
recipients:export
|
|
||||||
```
|
|
||||||
|
|
||||||
### Files
|
|
||||||
|
|
||||||
```text
|
|
||||||
files:read
|
|
||||||
files:download
|
|
||||||
files:upload
|
|
||||||
files:organize
|
|
||||||
files:share
|
|
||||||
files:delete
|
|
||||||
files:admin
|
|
||||||
```
|
|
||||||
|
|
||||||
### Reports and Audit
|
|
||||||
|
|
||||||
```text
|
|
||||||
reports:read
|
|
||||||
reports:export
|
|
||||||
reports:send
|
|
||||||
audit:read
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mail Servers
|
|
||||||
|
|
||||||
```text
|
|
||||||
mail_servers:read
|
|
||||||
mail_servers:use
|
|
||||||
mail_servers:test
|
|
||||||
mail_servers:write
|
|
||||||
mail_servers:manage_credentials
|
|
||||||
```
|
|
||||||
|
|
||||||
### Tenant Administration
|
|
||||||
|
|
||||||
```text
|
|
||||||
admin:users:read
|
|
||||||
admin:users:create
|
|
||||||
admin:users:update
|
|
||||||
admin:users:suspend
|
|
||||||
|
|
||||||
admin:groups:read
|
|
||||||
admin:groups:write
|
|
||||||
admin:groups:manage_members
|
|
||||||
|
|
||||||
admin:roles:read
|
|
||||||
admin:roles:write
|
|
||||||
admin:roles:assign
|
|
||||||
|
|
||||||
admin:api_keys:read
|
|
||||||
admin:api_keys:create
|
|
||||||
admin:api_keys:revoke
|
|
||||||
|
|
||||||
admin:settings:read
|
|
||||||
admin:settings:write
|
|
||||||
admin:policies:read
|
|
||||||
admin:policies:write
|
|
||||||
```
|
|
||||||
|
|
||||||
## Canonical System Permissions - 18
|
|
||||||
|
|
||||||
```text
|
|
||||||
system:tenants:read
|
|
||||||
system:tenants:create
|
|
||||||
system:tenants:update
|
|
||||||
system:tenants:suspend
|
|
||||||
|
|
||||||
system:accounts:read
|
|
||||||
system:accounts:create
|
|
||||||
system:accounts:update
|
|
||||||
system:accounts:suspend
|
|
||||||
|
|
||||||
system:roles:read
|
|
||||||
system:roles:write
|
|
||||||
system:roles:assign
|
|
||||||
|
|
||||||
system:access:read
|
|
||||||
system:access:assign
|
|
||||||
|
|
||||||
system:audit:read
|
|
||||||
system:settings:read
|
|
||||||
system:settings:write
|
|
||||||
system:governance:read
|
|
||||||
system:governance:write
|
|
||||||
```
|
|
||||||
|
|
||||||
`system:access:*` remains as a compatibility/read and assignment boundary for cross-tenant/system access handling. It is not a separate primary UI area.
|
|
||||||
|
|
||||||
## Default Tenant Roles
|
|
||||||
|
|
||||||
- **Owner:** `tenant:*`. At least one active operational owner must remain.
|
|
||||||
- **Tenant administrator:** settings, policies, users, groups, roles and API keys plus read access to campaigns/files/reports/audit. Real delivery remains separately delegable.
|
|
||||||
- **Administrator (legacy):** all tenant permissions for upgraded installations.
|
|
||||||
- **Access administrator:** membership and assignment management within delegation limits.
|
|
||||||
- **Campaign manager:** prepare, validate and build campaigns; no review approval or real delivery by default.
|
|
||||||
- **Reviewer:** inspect and approve prepared campaign messages.
|
|
||||||
- **Sender:** mock-test, queue, control, send, retry and reconcile prepared campaigns; can use/test approved mail profiles.
|
|
||||||
- **File manager:** managed file operations without campaign delivery rights.
|
|
||||||
- **Viewer:** read campaigns, recipients, files and reports.
|
|
||||||
- **Auditor:** read campaigns, recipient evidence, reports and audit records; export detailed evidence.
|
|
||||||
|
|
||||||
## Default System Roles
|
|
||||||
|
|
||||||
- **System owner:** `system:*`, protected. At least one active account must retain it.
|
|
||||||
- **System administrator:** all specific system permissions, editable and not protected.
|
|
||||||
- **System auditor:** read-only system registry/settings/governance/audit role, editable.
|
|
||||||
|
|
||||||
## Delegation Ceiling
|
|
||||||
|
|
||||||
For role definition, assignment and API-key creation:
|
|
||||||
|
|
||||||
```text
|
|
||||||
requested scopes subset of actor delegateable scopes
|
|
||||||
```
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
1. Tenant roles may contain tenant scopes only.
|
|
||||||
2. System roles may contain system scopes only.
|
|
||||||
3. Definition rights and assignment rights are separate.
|
|
||||||
4. Group definition and group membership management are separate.
|
|
||||||
5. API-key scopes are intersected with the owner's current effective scopes on every request.
|
|
||||||
6. Suspended accounts, users, tenants or groups stop contributing access immediately.
|
|
||||||
7. Administrative updates are field-sensitive; a user with only status authority cannot change role assignments.
|
|
||||||
|
|
||||||
## Campaign Ownership and ACLs
|
|
||||||
|
|
||||||
A campaign has exactly one owner:
|
|
||||||
|
|
||||||
```text
|
|
||||||
owner user OR owner group
|
|
||||||
```
|
|
||||||
|
|
||||||
Additional active shares may target users or groups with `read` or `write`.
|
|
||||||
|
|
||||||
Resolution:
|
|
||||||
|
|
||||||
- owner user: read and write;
|
|
||||||
- member of owner group: read and write;
|
|
||||||
- explicit read share: read;
|
|
||||||
- explicit write share: read and write;
|
|
||||||
- `tenant:*`: tenant-wide ACL bypass;
|
|
||||||
- ordinary campaign permission without ownership/share: no object access.
|
|
||||||
|
|
||||||
ACLs do not add capabilities. A write share still needs the specific permission for update, validation, review, send, report, retry or reconciliation.
|
|
||||||
|
|
||||||
## Sensitive Recipient Boundary
|
|
||||||
|
|
||||||
Recipient-complete campaign JSON, message data and job detail require `recipients:read`. Recipient edits require `recipients:write`; exports require `recipients:export`; import is reserved for the dedicated recipient import/list workflow.
|
|
||||||
|
|
||||||
## Files
|
|
||||||
|
|
||||||
| Permission | Operations |
|
|
||||||
|---|---|
|
|
||||||
| `files:read` | list, search, inspect, resolve metadata |
|
|
||||||
| `files:download` | download file bytes and generated ZIP archives |
|
|
||||||
| `files:upload` | upload files and ZIP contents |
|
|
||||||
| `files:organize` | create folders, rename, move, copy and bulk rename |
|
|
||||||
| `files:share` | create/revoke file shares |
|
|
||||||
| `files:delete` | delete/hide files and folders subject to retention |
|
|
||||||
| `files:admin` | tenant-wide administration of user/group file spaces |
|
|
||||||
|
|
||||||
## Mail Servers
|
|
||||||
|
|
||||||
| Permission | Boundary |
|
|
||||||
|---|---|
|
|
||||||
| `mail_servers:read` | profile metadata and effective policy visibility |
|
|
||||||
| `mail_servers:use` | select an approved profile without reading secrets |
|
|
||||||
| `mail_servers:test` | run server-side connection tests |
|
|
||||||
| `mail_servers:write` | define/edit profiles in allowed scopes |
|
|
||||||
| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or campaign-level credentials where policy allows |
|
|
||||||
|
|
||||||
Reusable encrypted profiles now exist. Effective usability is also constrained by hierarchical mail-profile policy, ownership, allowed/forced profile sets, credential inheritance mode, the lower-level override switch for that mode, and allow/deny patterns.
|
|
||||||
|
|
||||||
## Sessions, API Keys and CSRF
|
|
||||||
|
|
||||||
- Browser login creates an HttpOnly session cookie and a separate readable CSRF cookie.
|
|
||||||
- Unsafe cookie-authenticated requests require matching CSRF cookie/header and stored CSRF hash.
|
|
||||||
- API keys remain supported for CLI/automation and do not use browser CSRF.
|
|
||||||
- Login responses still expose a compatibility session token in the response body; the WebUI does not persist it.
|
|
||||||
|
|
||||||
## Legacy Compatibility
|
|
||||||
|
|
||||||
Runtime aliases remain only for names that are no longer canonical, including:
|
|
||||||
|
|
||||||
```text
|
|
||||||
campaign:write
|
|
||||||
attachments:read
|
|
||||||
attachments:write
|
|
||||||
admin:users
|
|
||||||
admin:users:write
|
|
||||||
admin:api_keys:write
|
|
||||||
admin:settings
|
|
||||||
system:tenants:write
|
|
||||||
system:access:write
|
|
||||||
```
|
|
||||||
|
|
||||||
Canonical scopes are not widened by runtime alias expansion after migration.
|
|
||||||
|
|
||||||
## Deferred Permission Families
|
|
||||||
|
|
||||||
Add these only with their corresponding implemented features:
|
|
||||||
|
|
||||||
```text
|
|
||||||
templates:*
|
|
||||||
address_books:*
|
|
||||||
recipient_lists:*
|
|
||||||
connectors:*
|
|
||||||
dsar:*
|
|
||||||
system:monitoring:read
|
|
||||||
system:backups:run
|
|
||||||
system:backups:restore
|
|
||||||
system:updates:apply
|
|
||||||
system:updates:rollback
|
|
||||||
```
|
|
||||||
|
|
||||||
A separate `retention:*` family is not currently canonical because retention is managed through system settings and tenant policy scopes. Add it only if retention operation duties need separation from general policy/settings administration.
|
|
||||||
@@ -1,129 +0,0 @@
|
|||||||
# Release Catalog Workflow
|
|
||||||
|
|
||||||
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
|
||||||
verified by `govoplan-core` before installer plans are accepted.
|
|
||||||
|
|
||||||
Private signing keys must stay outside all git repositories. Public keyrings
|
|
||||||
are published with the website.
|
|
||||||
|
|
||||||
## One-Time Key Setup
|
|
||||||
|
|
||||||
Create the first catalog signing key on the release machine:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
|
||||||
mkdir -p "$KEY_DIR"
|
|
||||||
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
|
||||||
--key-id release-key-1 \
|
|
||||||
--private-key "$KEY_DIR/release-key-1.pem" \
|
|
||||||
--public-key "$KEY_DIR/release-key-1.pub" \
|
|
||||||
--keyring "$KEY_DIR/catalog-keyring.json"
|
|
||||||
```
|
|
||||||
|
|
||||||
Keep `release-key-1.pem` private. The generated keyring contains only public
|
|
||||||
material.
|
|
||||||
|
|
||||||
## Publish Current Release Catalog
|
|
||||||
|
|
||||||
Generate the signed catalog into `govoplan-web`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
|
||||||
scripts/publish-release-catalog.sh \
|
|
||||||
--version 0.1.4 \
|
|
||||||
--sequence 202607071340 \
|
|
||||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
|
||||||
--build-web
|
|
||||||
```
|
|
||||||
|
|
||||||
This writes:
|
|
||||||
|
|
||||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
|
||||||
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
|
||||||
|
|
||||||
The wrapper validates the catalog with core using the generated public keyring.
|
|
||||||
|
|
||||||
## Publish After A New Release
|
|
||||||
|
|
||||||
For normal module/core releases, first tag and push the module/core repos. Then
|
|
||||||
publish the website catalog:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
|
||||||
scripts/publish-release-catalog.sh \
|
|
||||||
--version <x.y.z> \
|
|
||||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
|
||||||
--build-web \
|
|
||||||
--commit \
|
|
||||||
--tag \
|
|
||||||
--push
|
|
||||||
```
|
|
||||||
|
|
||||||
The website tag is `catalog-v<x.y.z>`. The public URL is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The public keyring URL is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integrated Release Script
|
|
||||||
|
|
||||||
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
|
||||||
tags have been pushed:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/DATA/git/govoplan-core
|
|
||||||
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
|
||||||
scripts/push-release-tag.sh \
|
|
||||||
--bump subversion \
|
|
||||||
--publish-web-catalog \
|
|
||||||
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
|
||||||
--build-web-catalog
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--catalog-signing-key` more than once during a key rotation window. The
|
|
||||||
catalog will contain multiple signatures and the public keyring will include the
|
|
||||||
corresponding public keys.
|
|
||||||
|
|
||||||
## Consumer Configuration
|
|
||||||
|
|
||||||
On a GovOPlaN installation that should consume the official stable catalog:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
For production, copy the public keyring into deployment configuration and pin it
|
|
||||||
locally. Do not rely on a URL-fetched keyring as the only trust root.
|
|
||||||
|
|
||||||
## Core Updates
|
|
||||||
|
|
||||||
`stable.json` includes a top-level `core_release` section for operator/update
|
|
||||||
tooling. Core is intentionally not listed as a normal module entry, because it
|
|
||||||
must not be added to saved enabled-module state. Core upgrades should remain an
|
|
||||||
operator-supervised package update with restart and health checks.
|
|
||||||
|
|
||||||
## Key Rotation
|
|
||||||
|
|
||||||
1. Generate the next private key outside git.
|
|
||||||
2. Run `publish-release-catalog.sh` with both signing keys.
|
|
||||||
3. Publish the web catalog/keyring.
|
|
||||||
4. Roll the new public keyring into installations.
|
|
||||||
5. Stop signing with the old key after the supported fleet trusts the new key.
|
|
||||||
6. Mark compromised keys as revoked in the public keyring and publish a higher
|
|
||||||
sequence catalog signed by a trusted uncompromised key.
|
|
||||||
|
|
||||||
@@ -1,8 +1,17 @@
|
|||||||
# GovOPlaN Release Dependencies
|
# GovOPlaN Release Dependencies
|
||||||
|
|
||||||
Release installs must not depend on sibling checkout paths. Local development can keep editable installs and `file:` WebUI links, but release packaging should resolve modules from tagged git refs or from a package registry.
|
This document owns release package composition, signed package catalogs,
|
||||||
|
license checks, catalog publishing, migration baselines, and the final release
|
||||||
|
checklist.
|
||||||
|
|
||||||
## Backend
|
Operator runtime configuration and module install/uninstall execution live in
|
||||||
|
`DEPLOYMENT_OPERATOR_GUIDE.md`.
|
||||||
|
|
||||||
|
## Backend Packages
|
||||||
|
|
||||||
|
Release installs must not depend on sibling checkout paths. Local development
|
||||||
|
can keep editable installs and `file:` WebUI links, but release packaging must
|
||||||
|
resolve modules from tagged git refs or from a package registry.
|
||||||
|
|
||||||
Local development:
|
Local development:
|
||||||
|
|
||||||
@@ -18,280 +27,41 @@ cd /mnt/DATA/git/govoplan-core
|
|||||||
./.venv/bin/python -m pip install -r requirements-release.txt
|
./.venv/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
`.[server]` is resolved relative to the current working directory. If you create the virtualenv elsewhere, still run the install command from the core checkout:
|
`.[server]` is resolved relative to the current working directory. If you
|
||||||
|
create the virtualenv elsewhere, still run the install command from the core
|
||||||
|
checkout:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
/tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
`requirements-release.txt` pins the module repositories to the release tag. Update those refs when cutting a release:
|
`requirements-release.txt` pins the module repositories to the release tag.
|
||||||
|
Update those refs when cutting a release:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.4
|
govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.6
|
||||||
govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.4
|
govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.6
|
||||||
govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.4
|
govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.6
|
||||||
govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.4
|
govoplan-organizations git@git.add-ideas.de:add-ideas/govoplan-organizations.git v0.1.6
|
||||||
govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.4
|
govoplan-identity git@git.add-ideas.de:add-ideas/govoplan-identity.git v0.1.6
|
||||||
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.4
|
govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.6
|
||||||
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.4
|
govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.6
|
||||||
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.4
|
govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.6
|
||||||
govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.4
|
govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.6
|
||||||
|
govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.6
|
||||||
|
govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.6
|
||||||
```
|
```
|
||||||
|
|
||||||
## Runtime Module Package Changes
|
## WebUI Packages
|
||||||
|
|
||||||
The admin module manager can hot-enable and hot-disable packages that are
|
Local development uses `webui/package.json`, which may point at sibling module
|
||||||
already installed. It does not install or uninstall Python/npm packages from
|
checkouts while active development is happening.
|
||||||
inside the running server.
|
|
||||||
|
|
||||||
For runtime package changes, create an operator install plan in Admin > System >
|
Release WebUI installs should use `webui/package.release.json`. It points
|
||||||
Modules. The module manager shows the trusted installer preflight status and
|
module dependencies at the same tagged git repositories. After the module tags
|
||||||
blocks unsafe uninstalls before the operator touches packages.
|
referenced there exist, generate the committed release lockfile without
|
||||||
|
touching the development package files:
|
||||||
Preflight from the server shell:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --format shell
|
|
||||||
```
|
|
||||||
|
|
||||||
Apply from a controlled operator shell while maintenance mode is active:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --apply --build-webui
|
|
||||||
```
|
|
||||||
|
|
||||||
For real install/uninstall work, prefer supervised mode with the deployment's
|
|
||||||
restart command and health endpoint:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer \
|
|
||||||
--supervise \
|
|
||||||
--migrate \
|
|
||||||
--build-webui \
|
|
||||||
--health-url http://127.0.0.1:8000/health \
|
|
||||||
--restart-command '<restart govoplan server>'
|
|
||||||
```
|
|
||||||
|
|
||||||
To let the admin UI trigger package work without executing pip/npm inside a
|
|
||||||
FastAPI request, run the installer daemon in a separate operator shell. This is
|
|
||||||
the preferred development/early-production mode for now because the operator can
|
|
||||||
watch output, stop before queueing risky changes, and keep restart commands
|
|
||||||
deployment-specific:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer \
|
|
||||||
--daemon \
|
|
||||||
--migrate \
|
|
||||||
--build-webui \
|
|
||||||
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
|
||||||
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
|
||||||
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
|
||||||
--health-url http://127.0.0.1:8000/health \
|
|
||||||
--restart-command '<restart govoplan server>'
|
|
||||||
```
|
|
||||||
|
|
||||||
Admin > System > Modules can then queue the saved install plan as a supervised
|
|
||||||
request. Install rows can be planned directly from the approved package catalog;
|
|
||||||
uninstall rows are generated from installed, disabled modules so the Python
|
|
||||||
distribution and WebUI package names do not need to be typed by hand. The
|
|
||||||
daemon claims one queued request at a time and writes request/run records below
|
|
||||||
`runtime/module-installer`. For process-manager one-shot usage or tests, use
|
|
||||||
`--daemon-once`. The daemon also writes
|
|
||||||
`runtime/module-installer/daemon.status.json`; check it with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --daemon-status --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
The installer uses a runtime lock, snapshots `pip freeze` plus WebUI
|
|
||||||
`package.json`/`package-lock.json`, writes a run record below
|
|
||||||
`runtime/module-installer/runs`, and marks planned rows as applied only after
|
|
||||||
all commands succeed. When `--migrate` is used with a `sqlite:///` database URL,
|
|
||||||
the installer also snapshots the SQLite database with SQLite's backup API before
|
|
||||||
running migrations. For other database engines, pass external backup/restore
|
|
||||||
hooks:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer \
|
|
||||||
--supervise \
|
|
||||||
--migrate \
|
|
||||||
--database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
|
||||||
--database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \
|
|
||||||
--database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \
|
|
||||||
--health-url http://127.0.0.1:8000/health \
|
|
||||||
--restart-command '<restart govoplan server>'
|
|
||||||
```
|
|
||||||
|
|
||||||
The backup command runs before migrations. The restore-check command validates
|
|
||||||
the produced backup artifact before migrations proceed, without restoring over
|
|
||||||
the live database. The restore command is stored in the run record and runs
|
|
||||||
during rollback unless an override is passed to `--rollback`.
|
|
||||||
|
|
||||||
Supervised mode treats package command failure, migration failure, restart
|
|
||||||
failure, and health timeout as rollback triggers. It restores the Python/WebUI
|
|
||||||
package snapshots, re-runs the restart command when supplied, and restores the
|
|
||||||
saved install plan state so the operator can correct it. The supervisor must
|
|
||||||
run outside the FastAPI server process; the admin UI saves and validates plans
|
|
||||||
but does not mutate packages from an HTTP request.
|
|
||||||
|
|
||||||
After a successful install plan, the installer adds installed modules to saved
|
|
||||||
startup state by default so the restarted server can discover and enable them.
|
|
||||||
After a successful uninstall plan, the installer removes uninstalled modules
|
|
||||||
from saved startup state by default. Use
|
|
||||||
`--no-activate-installed-modules` or
|
|
||||||
`--keep-uninstalled-modules-in-desired` only for staged rollout workflows that
|
|
||||||
will update module state separately.
|
|
||||||
|
|
||||||
Uninstall is non-destructive by default. A planned uninstall row can set
|
|
||||||
`destroy_data: true` to request destructive module retirement. The module must
|
|
||||||
provide an automated retirement provider, and the installer snapshots the
|
|
||||||
database before dropping module-owned tables. For SQLite this uses the built-in
|
|
||||||
snapshot path; for PostgreSQL or another non-SQLite database, provide
|
|
||||||
`--database-backup-command`, `--database-restore-check-command`, and
|
|
||||||
`--database-restore-command`. If a destructive run fails during package removal,
|
|
||||||
the installer restores the database snapshot before returning the failed run
|
|
||||||
result; supervised restart/health failures also roll back through the normal
|
|
||||||
supervisor path.
|
|
||||||
|
|
||||||
Package rollback is automatic. SQLite database rollback is automatic for
|
|
||||||
installer runs that used `--migrate` and captured a database snapshot.
|
|
||||||
Non-SQLite rollback is automatic when the run used
|
|
||||||
`--database-backup-command` and `--database-restore-command`; otherwise migrated
|
|
||||||
non-SQLite runs are blocked before package changes are applied.
|
|
||||||
|
|
||||||
Rollback uses the saved run snapshot:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --rollback <run-id>
|
|
||||||
govoplan-module-installer --rollback <run-id> --database-restore-command '<override restore command>'
|
|
||||||
```
|
|
||||||
|
|
||||||
Database hook commands run with these environment variables:
|
|
||||||
|
|
||||||
- `GOVOPLAN_INSTALLER_RUN_DIR`: the run snapshot directory
|
|
||||||
- `GOVOPLAN_DATABASE_URL`: the configured database URL
|
|
||||||
- `GOVOPLAN_DATABASE_BACKUP_PATH`: a suggested backup artifact path inside the
|
|
||||||
run directory
|
|
||||||
- `GOVOPLAN_DATABASE_BACKUP_METADATA`: optional JSON metadata path that backup
|
|
||||||
commands may write for operator diagnostics
|
|
||||||
|
|
||||||
Avoid embedding secrets directly in commands; prefer environment variables,
|
|
||||||
service credentials, or deployment-local secret injection.
|
|
||||||
|
|
||||||
Inspect installer history and lock state from the operator shell:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --list-runs --format json
|
|
||||||
govoplan-module-installer --show-run <run-id> --format json
|
|
||||||
govoplan-module-installer --lock-status --format json
|
|
||||||
govoplan-module-installer --list-requests --format json
|
|
||||||
govoplan-module-installer --show-request <request-id> --format json
|
|
||||||
govoplan-module-installer --cancel-request <request-id> --format json
|
|
||||||
govoplan-module-installer --retry-request <request-id> --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Package catalogs can be local files or remote static resources, for example
|
|
||||||
served by `govoplan-web`. Set `GOVOPLAN_MODULE_PACKAGE_CATALOG` for a local file
|
|
||||||
or `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` for a remote catalog matching
|
|
||||||
`docs/module-package-catalog.example.json`; the admin UI will show those entries
|
|
||||||
and can save them into the install plan. This keeps the release approval
|
|
||||||
decision outside the running server while avoiding hand-typed package refs.
|
|
||||||
Remote catalogs can be cached for offline inspection:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Validate the catalog before handing it to operators:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer --validate-package-catalog docs/module-package-catalog.example.json --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Release catalogs should be signed, channel-gated, expiring, and sequence
|
|
||||||
tracked. The supported signing format is an Ed25519 signature over the
|
|
||||||
canonical catalog JSON object with the `signature` and `signatures` fields
|
|
||||||
removed. Core accepts the legacy single `signature` field and the newer
|
|
||||||
`signatures` array used during key rotation. Sign a catalog with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer \
|
|
||||||
--sign-package-catalog docs/module-package-catalog.example.json \
|
|
||||||
--catalog-signing-key-id release-key-1 \
|
|
||||||
--catalog-signing-private-key /path/to/ed25519-private.pem
|
|
||||||
```
|
|
||||||
|
|
||||||
Validate an approved release catalog with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
govoplan-module-installer \
|
|
||||||
--validate-package-catalog docs/module-package-catalog.example.json \
|
|
||||||
--require-signed-catalog \
|
|
||||||
--approved-catalog-channel stable \
|
|
||||||
--catalog-trusted-key release-key-1=<base64-ed25519-public-key> \
|
|
||||||
--format json
|
|
||||||
```
|
|
||||||
|
|
||||||
For the admin UI/daemon path, configure the same policy through environment
|
|
||||||
variables:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG=/path/to/catalog.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":"<base64-ed25519-public-key>"}'
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
|
||||||
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
|
||||||
```
|
|
||||||
|
|
||||||
Trusted keys can also be loaded from
|
|
||||||
`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE`. A URL-backed keyring is
|
|
||||||
supported for development and tightly controlled deployments through
|
|
||||||
`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL` plus
|
|
||||||
`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE`, but production systems
|
|
||||||
should pin trusted keys locally.
|
|
||||||
|
|
||||||
Catalog entries can declare `license_features`. If
|
|
||||||
`GOVOPLAN_LICENSE_ENFORCEMENT=true`, core blocks planning catalog installs
|
|
||||||
whose required features are not present in the configured offline license:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json
|
|
||||||
GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json
|
|
||||||
GOVOPLAN_LICENSE_ENFORCEMENT=true
|
|
||||||
```
|
|
||||||
|
|
||||||
See `docs/CATALOG_TRUST_AND_LICENSING.md` for key rotation, replay protection,
|
|
||||||
and licensing details. See `docs/RELEASE_CATALOG_WORKFLOW.md` for the concrete
|
|
||||||
release-machine workflow that generates signing keys and publishes signed
|
|
||||||
catalogs through `govoplan-web`. Unsigned catalogs remain usable for local
|
|
||||||
development when signature enforcement is off, but the admin UI labels them as
|
|
||||||
unsigned.
|
|
||||||
|
|
||||||
Install rows must use tagged package/git refs or registry packages, not local
|
|
||||||
`file:` or workspace links. The installer daemon can run `npm install` and
|
|
||||||
`npm run build` for WebUI package changes; that is the supported path. Browser
|
|
||||||
remote bundles are still experimental and should be treated as a controlled
|
|
||||||
deployment option, not the normal install/uninstall mechanism.
|
|
||||||
|
|
||||||
Module manifests can declare core compatibility bounds and uninstall guard
|
|
||||||
providers. Preflight blocks incompatible manifest contracts/core versions,
|
|
||||||
active modules, desired startup state, protected modules, and active dependents.
|
|
||||||
Default uninstall is non-destructive: module data and schema remain dormant if
|
|
||||||
the package is removed. Persistent-data guards therefore warn by default instead
|
|
||||||
of requiring export/delete. A module that supports explicit data/schema
|
|
||||||
retirement should register a retirement provider. When `destroy_data` is set on
|
|
||||||
an uninstall plan row, that provider is allowed to destroy module-owned data
|
|
||||||
after the installer has captured a database snapshot; otherwise it is used only
|
|
||||||
for preflight reporting.
|
|
||||||
|
|
||||||
## WebUI
|
|
||||||
|
|
||||||
Local development uses `webui/package.json`, which may point at sibling module checkouts while active development is happening.
|
|
||||||
|
|
||||||
Release WebUI installs should use `webui/package.release.json`. It points module dependencies at the same tagged git repositories. After the module tags referenced there exist, generate the committed release lockfile without touching the development package files:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
@@ -300,16 +70,48 @@ cd webui
|
|||||||
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`, `@govoplan/files-webui`, `@govoplan/mail-webui`, `@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository roots even though their source lives below `webui/src`.
|
The module repositories include root-level npm package manifests so git
|
||||||
|
installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`,
|
||||||
|
`@govoplan/files-webui`, `@govoplan/mail-webui`,
|
||||||
|
`@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository
|
||||||
|
roots even though their source lives below `webui/src`.
|
||||||
|
|
||||||
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps or accepts the target version, updates Python/WebUI/module manifest versions, commits/tags/pushes the module repositories first, regenerates `webui/package-lock.release.json`, and then commits/tags/pushes core. If the working tree has already been bumped, pass the current version explicitly:
|
### Release Lockfile Strategy
|
||||||
|
|
||||||
|
The supported release composition currently is the full GovOPlaN product: core
|
||||||
|
plus access, admin, tenancy, organizations, identity, policy, audit,
|
||||||
|
dashboard, files, mail, campaign, calendar, docs, and ops. Keep one committed
|
||||||
|
full-product release lockfile at
|
||||||
|
`webui/package-lock.release.json`, generated from
|
||||||
|
`webui/package.release.json` in a clean release workspace. Development
|
||||||
|
`package-lock.json` may continue to point at local `file:` dependencies.
|
||||||
|
|
||||||
|
Frontend module permutations are regression-tested through
|
||||||
|
`GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through
|
||||||
|
committed lockfiles for every possible combination. If a smaller composition
|
||||||
|
becomes a separately shipped product, add an explicit release manifest and
|
||||||
|
lockfile pair for that product, for example
|
||||||
|
`package.release.files-mail.json` and `package-lock.release.files-mail.json`,
|
||||||
|
generated in a clean release workspace from tagged git dependencies.
|
||||||
|
|
||||||
|
## Release Tag Script
|
||||||
|
|
||||||
|
The normal release path is automated by `scripts/push-release-tag.sh`: it bumps
|
||||||
|
or accepts the target version, updates Python/WebUI/module manifest versions,
|
||||||
|
commits/tags/pushes the module repositories first, regenerates
|
||||||
|
`webui/package-lock.release.json`, and then commits/tags/pushes core. If the
|
||||||
|
working tree has already been bumped, pass the current version explicitly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/DATA/git/govoplan-core
|
cd /mnt/DATA/git/govoplan-core
|
||||||
scripts/push-release-tag.sh --version 0.1.2
|
scripts/push-release-tag.sh --version 0.1.6
|
||||||
```
|
```
|
||||||
|
|
||||||
The script also includes GovOPlaN roadmap/scaffold module repositories that do not yet have package metadata. Those repositories are committed, tagged, and pushed with the same release tag, but they are tag-only until they contain `pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories are not listed in `requirements-release.txt` or `webui/package.release.json`.
|
The script also includes GovOPlaN roadmap/scaffold module repositories that do
|
||||||
|
not yet have package metadata. Those repositories are committed, tagged, and
|
||||||
|
pushed with the same release tag, but they are tag-only until they contain
|
||||||
|
`pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories
|
||||||
|
are not listed in `requirements-release.txt` or `webui/package.release.json`.
|
||||||
|
|
||||||
Current tag-only module repositories:
|
Current tag-only module repositories:
|
||||||
|
|
||||||
@@ -325,7 +127,6 @@ Current tag-only module repositories:
|
|||||||
- `govoplan-idm`
|
- `govoplan-idm`
|
||||||
- `govoplan-ledger`
|
- `govoplan-ledger`
|
||||||
- `govoplan-notifications`
|
- `govoplan-notifications`
|
||||||
- `govoplan-ops`
|
|
||||||
- `govoplan-payments`
|
- `govoplan-payments`
|
||||||
- `govoplan-portal`
|
- `govoplan-portal`
|
||||||
- `govoplan-reporting`
|
- `govoplan-reporting`
|
||||||
@@ -338,17 +139,486 @@ Current tag-only module repositories:
|
|||||||
- `govoplan-xrechnung`
|
- `govoplan-xrechnung`
|
||||||
- `govoplan-xta-osci`
|
- `govoplan-xta-osci`
|
||||||
|
|
||||||
### Release lockfile strategy
|
## Catalog Trust And Licensing
|
||||||
|
|
||||||
The supported release composition currently is the full Multi Seal Mail product: core plus access, admin, tenancy, policy, audit, files, mail, campaign, and calendar. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies.
|
GovOPlaN module install and uninstall must remain operator-controlled. The
|
||||||
|
running server may plan and validate package changes, but package mutation is
|
||||||
|
performed by the separate installer daemon or an operator shell during
|
||||||
|
maintenance mode.
|
||||||
|
|
||||||
Frontend module permutations are regression-tested through `GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through committed lockfiles for every possible combination. If a smaller composition becomes a separately shipped product, add an explicit release manifest and lockfile pair for that product, for example `package.release.files-mail.json` and `package-lock.release.files-mail.json`, generated in a clean release workspace from tagged git dependencies.
|
`govoplan-web` is the public static distribution surface for official catalog
|
||||||
|
resources:
|
||||||
|
|
||||||
|
- signed module package catalogs, grouped by release channel
|
||||||
|
- public catalog keyrings
|
||||||
|
- public license verification keyrings
|
||||||
|
- examples and operator-facing download paths
|
||||||
|
|
||||||
|
`govoplan-core` is the verifier and orchestrator:
|
||||||
|
|
||||||
|
- fetches a local or remote module catalog
|
||||||
|
- verifies catalog signatures against configured trusted keys
|
||||||
|
- enforces approved release channels
|
||||||
|
- rejects expired or not-yet-valid catalogs
|
||||||
|
- records accepted catalog sequence numbers for replay protection
|
||||||
|
- checks catalog entry license feature requirements before planning installs
|
||||||
|
- writes installer plans and request records
|
||||||
|
|
||||||
|
Feature and platform modules own their package artifacts, manifests, migration
|
||||||
|
metadata, retirement providers, and optional lifecycle behavior.
|
||||||
|
|
||||||
|
Core accepts either a local catalog file or a remote URL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
||||||
|
```
|
||||||
|
|
||||||
|
If both file and URL are set, the URL wins. The cache is used when a remote
|
||||||
|
fetch fails, so an operator can still inspect the last known catalog. A cached
|
||||||
|
catalog must still pass signature, freshness, channel, and replay validation.
|
||||||
|
|
||||||
|
An official catalog is a JSON object with:
|
||||||
|
|
||||||
|
- `catalog_version`
|
||||||
|
- `channel`
|
||||||
|
- `sequence`
|
||||||
|
- `generated_at`
|
||||||
|
- `not_before` when delayed activation is needed
|
||||||
|
- `expires_at`
|
||||||
|
- `modules`
|
||||||
|
- `signatures`
|
||||||
|
|
||||||
|
Each module entry can declare:
|
||||||
|
|
||||||
|
- backend package name and pinned install reference
|
||||||
|
- WebUI package name and pinned install reference
|
||||||
|
- display metadata and tags
|
||||||
|
- `license_features`, the feature entitlements required to plan that install
|
||||||
|
|
||||||
|
The signature is Ed25519 over canonical JSON with both `signature` and
|
||||||
|
`signatures` removed. Core accepts the legacy single `signature` field and the
|
||||||
|
new `signatures` array.
|
||||||
|
|
||||||
|
Trusted catalog keys are configured locally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":"<base64 public key>"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
For development or tightly controlled deployments, a keyring can be read from a
|
||||||
|
URL and cached:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Production installations should pin the trusted keyring locally or ship it
|
||||||
|
through deployment configuration. Fetching trusted keys from the same public
|
||||||
|
origin as the catalog is convenient, but that origin must not become the only
|
||||||
|
trust root.
|
||||||
|
|
||||||
|
## Dependency Audits
|
||||||
|
|
||||||
|
Dependency vulnerability checks are documented in
|
||||||
|
[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same
|
||||||
|
check against release dependency refs on pushes, pull requests, and a weekly
|
||||||
|
schedule.
|
||||||
|
|
||||||
|
Keyring entries support:
|
||||||
|
|
||||||
|
- `key_id`
|
||||||
|
- `public_key` or `public_key_base64`
|
||||||
|
- `status`: `active`, `next`, `retired`, `revoked`, or `disabled`
|
||||||
|
- `not_before`
|
||||||
|
- `not_after`
|
||||||
|
|
||||||
|
Rotation process:
|
||||||
|
|
||||||
|
1. Add the next public key to the local trusted keyring with status `next`.
|
||||||
|
2. Publish catalogs signed by both current and next keys.
|
||||||
|
3. Upgrade installations so the next key is locally trusted.
|
||||||
|
4. Promote the next key to `active`.
|
||||||
|
5. Retire the old key only after every supported installation trusts the new
|
||||||
|
key.
|
||||||
|
6. Mark a compromised key `revoked` and publish a higher sequence catalog
|
||||||
|
signed by an uncompromised key.
|
||||||
|
|
||||||
|
Use replay state in production:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Core records the accepted sequence per channel after a catalog entry is planned
|
||||||
|
from the admin interface. With strict sequence enforcement, a previously
|
||||||
|
accepted sequence is rejected; without strict enforcement, only older sequences
|
||||||
|
are rejected. Catalogs should always expire.
|
||||||
|
|
||||||
|
The sequence state file is operational state, not a trust root. Keep it on
|
||||||
|
persistent storage and include it in normal backups:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"channels": {
|
||||||
|
"stable": {
|
||||||
|
"last_sequence": 42,
|
||||||
|
"accepted_at": "2026-07-07T12:00:00Z",
|
||||||
|
"key_id": "release-key-1",
|
||||||
|
"source": "https://govoplan.example/catalogs/v1/channels/stable.json"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If the file is lost, restore it from backup. If no backup exists, reconstruct
|
||||||
|
each channel from the highest sequence already accepted in installer run
|
||||||
|
records, release records, or the currently deployed module package set. Do not
|
||||||
|
lower `last_sequence` to make an older catalog pass; publish a new higher
|
||||||
|
sequence catalog when the accepted point is uncertain.
|
||||||
|
|
||||||
|
If the file is corrupted, copy it aside for incident review, validate the
|
||||||
|
current signed catalog with channel and freshness enforcement, then rewrite the
|
||||||
|
state with the known accepted sequence. Keep
|
||||||
|
`GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true` and approved-channel
|
||||||
|
checks enabled during recovery. Temporarily disabling
|
||||||
|
`GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE` allows revalidating the same
|
||||||
|
sequence, but older sequences remain rejected once the reconstructed
|
||||||
|
`last_sequence` is in place.
|
||||||
|
|
||||||
|
Approved channels are deployment policy:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
The admin UI can display other catalog metadata, but core rejects catalogs from
|
||||||
|
unapproved channels when validation is configured.
|
||||||
|
|
||||||
|
Catalog entries can require license features:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"license_features": ["module.mail", "support.standard"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Core checks those requirements against an offline license file before allowing
|
||||||
|
the entry into the install plan.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json
|
||||||
|
GOVOPLAN_LICENSE_ENFORCEMENT=true
|
||||||
|
GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
License files are JSON objects with:
|
||||||
|
|
||||||
|
- `license_id`
|
||||||
|
- `subject`
|
||||||
|
- `features`
|
||||||
|
- `valid_from`
|
||||||
|
- `valid_until`
|
||||||
|
- `signature`
|
||||||
|
|
||||||
|
Issue or renew a license from an operator/release shell that has the Ed25519
|
||||||
|
private key:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--issue-license /srv/govoplan/license.json \
|
||||||
|
--license-id customer-2026-07 \
|
||||||
|
--license-subject "Example Municipality" \
|
||||||
|
--license-feature module.mail \
|
||||||
|
--license-feature support.standard \
|
||||||
|
--license-valid-until 2027-07-31T23:59:59Z \
|
||||||
|
--license-signing-key-id license-issuer-1 \
|
||||||
|
--license-signing-private-key /srv/govoplan/secrets/license-issuer-1.pem \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate an imported license without exposing secrets:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
govoplan-module-installer \
|
||||||
|
--validate-license /srv/govoplan/license.json \
|
||||||
|
--license-trusted-key license-issuer-1="<base64 public key>" \
|
||||||
|
--require-trusted-license \
|
||||||
|
--license-required-feature module.mail \
|
||||||
|
--format json
|
||||||
|
```
|
||||||
|
|
||||||
|
The CLI and admin module catalog panel report the license id, subject,
|
||||||
|
validity window, signing key id, signed/trusted state, available features, and
|
||||||
|
missing entitlements for the configured package catalog. They do not expose
|
||||||
|
private signing material.
|
||||||
|
|
||||||
|
License enforcement can run in observe-only mode by leaving
|
||||||
|
`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license
|
||||||
|
data is surfaced as a warning but does not block planning.
|
||||||
|
|
||||||
|
Renewal is an ordinary re-issuance with a new `license_id`, extended
|
||||||
|
`valid_until`, and the full intended feature set. Import the renewed JSON to
|
||||||
|
`GOVOPLAN_LICENSE_FILE`, keep the previous file for audit, and validate it
|
||||||
|
before setting enforcement.
|
||||||
|
|
||||||
|
Revocation is handled through the trusted license keyring. Mark a compromised
|
||||||
|
or invalid issuer key as `revoked` or `disabled`, publish or deploy the updated
|
||||||
|
keyring, then reissue affected licenses with an active key. Installations that
|
||||||
|
run with `GOVOPLAN_LICENSE_ENFORCEMENT=true` reject licenses signed only by a
|
||||||
|
revoked key after the local keyring is updated.
|
||||||
|
|
||||||
|
Emergency fallback is deliberately explicit. Operators can temporarily unset
|
||||||
|
`GOVOPLAN_LICENSE_ENFORCEMENT` to keep package planning observable while a
|
||||||
|
license or keyring is recovered. Record the change in the operational incident
|
||||||
|
log, keep catalog signature and channel enforcement enabled, and restore
|
||||||
|
license enforcement after a trusted renewal validates successfully.
|
||||||
|
|
||||||
|
Licensing is intentionally separate from open-source code licensing. The
|
||||||
|
catalog/license mechanism can govern support channels, official release
|
||||||
|
eligibility, hosted update access, professional support, or commercial
|
||||||
|
entitlements without changing the source license of the repositories.
|
||||||
|
|
||||||
|
Production-grade distribution still needs remote registry/git artifact
|
||||||
|
resolution before package-manager apply, a hardened catalog publishing pipeline
|
||||||
|
in `govoplan-web`, and automated key rotation and emergency revocation drills.
|
||||||
|
|
||||||
|
## Release Catalog Publishing
|
||||||
|
|
||||||
|
GovOPlaN release catalogs are published by `govoplan-web` as static JSON and
|
||||||
|
verified by `govoplan-core` before installer plans are accepted. Private signing
|
||||||
|
keys must stay outside all git repositories. Public keyrings are published with
|
||||||
|
the website.
|
||||||
|
|
||||||
|
Create the first catalog signing key on the release machine:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
mkdir -p "$KEY_DIR"
|
||||||
|
./.venv/bin/python scripts/generate-catalog-keypair.py \
|
||||||
|
--key-id release-key-1 \
|
||||||
|
--private-key "$KEY_DIR/release-key-1.pem" \
|
||||||
|
--public-key "$KEY_DIR/release-key-1.pub" \
|
||||||
|
--keyring "$KEY_DIR/catalog-keyring.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
Keep `release-key-1.pem` private. The generated keyring contains only public
|
||||||
|
material.
|
||||||
|
|
||||||
|
Generate the signed catalog into `govoplan-web`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/publish-release-catalog.sh \
|
||||||
|
--version <x.y.z> \
|
||||||
|
--sequence 202607071340 \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web
|
||||||
|
```
|
||||||
|
|
||||||
|
This writes:
|
||||||
|
|
||||||
|
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json`
|
||||||
|
- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json`
|
||||||
|
|
||||||
|
The wrapper validates the catalog with core using the generated public keyring.
|
||||||
|
|
||||||
|
For normal module/core releases, first audit and record migration baselines,
|
||||||
|
then tag and push the module/core repos. Finally publish the website catalog:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/publish-release-catalog.sh \
|
||||||
|
--version <x.y.z> \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web \
|
||||||
|
--commit \
|
||||||
|
--tag \
|
||||||
|
--push
|
||||||
|
```
|
||||||
|
|
||||||
|
The website tag is `catalog-v<x.y.z>`. The public URL is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The public keyring URL is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://govoplan.add-ideas.de/catalogs/v1/keyring.json
|
||||||
|
```
|
||||||
|
|
||||||
|
`scripts/push-release-tag.sh` can publish the web catalog after module and core
|
||||||
|
tags have been pushed. It runs the migration release audit in automatic mode:
|
||||||
|
warning-only before the first recorded migration baseline, strict after a
|
||||||
|
baseline exists. Add `--strict-migration-audit` when you want to force strict
|
||||||
|
mode explicitly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
KEY_DIR="$HOME/.config/govoplan/release-keys"
|
||||||
|
scripts/push-release-tag.sh \
|
||||||
|
--bump subversion \
|
||||||
|
--strict-migration-audit \
|
||||||
|
--publish-web-catalog \
|
||||||
|
--catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \
|
||||||
|
--build-web-catalog
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--catalog-signing-key` more than once during a key rotation window. The
|
||||||
|
catalog will contain multiple signatures and the public keyring will include the
|
||||||
|
corresponding public keys.
|
||||||
|
|
||||||
|
On a GovOPlaN installation that should consume the official stable catalog:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json
|
||||||
|
GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true
|
||||||
|
```
|
||||||
|
|
||||||
|
For production, copy the public keyring into deployment configuration and pin it
|
||||||
|
locally. Do not rely on a URL-fetched keyring as the only trust root.
|
||||||
|
|
||||||
|
`stable.json` includes a top-level `core_release` section for operator/update
|
||||||
|
tooling. Core is intentionally not listed as a normal module entry because it
|
||||||
|
must not be added to saved enabled-module state. Core upgrades should remain an
|
||||||
|
operator-supervised package update with restart and health checks.
|
||||||
|
|
||||||
|
Key rotation for published catalogs:
|
||||||
|
|
||||||
|
1. Generate the next private key outside git.
|
||||||
|
2. Run `publish-release-catalog.sh` with both signing keys.
|
||||||
|
3. Publish the web catalog/keyring.
|
||||||
|
4. Roll the new public keyring into installations.
|
||||||
|
5. Stop signing with the old key after the supported fleet trusts the new key.
|
||||||
|
6. Mark compromised keys as revoked in the public keyring and publish a higher
|
||||||
|
sequence catalog signed by a trusted uncompromised key.
|
||||||
|
|
||||||
|
## PostgreSQL Release Check
|
||||||
|
|
||||||
|
Release candidates should pass a disposable PostgreSQL migration and startup
|
||||||
|
smoke check before tagging or publishing catalogs. Start the local testbed,
|
||||||
|
then run the permutation check from the core checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core/dev/postgres
|
||||||
|
cp .env.example .env
|
||||||
|
docker compose --env-file .env up -d
|
||||||
|
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
set -a
|
||||||
|
. dev/postgres/.env
|
||||||
|
set +a
|
||||||
|
./.venv/bin/python scripts/postgres-integration-check.py \
|
||||||
|
--database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \
|
||||||
|
--reset-schema
|
||||||
|
```
|
||||||
|
|
||||||
|
The script checks migrations and `/health` startup for core-only, files-only,
|
||||||
|
mail-only, campaign-only, campaign+files, campaign+mail, and full-product
|
||||||
|
module sets. `--reset-schema` is destructive and must only be used against a
|
||||||
|
throwaway database.
|
||||||
|
|
||||||
|
## Migration Baselines
|
||||||
|
|
||||||
|
Development migrations may be small and numerous while a feature is moving.
|
||||||
|
Before a stable release, unreleased migrations may be rewritten or squashed into
|
||||||
|
a release-level baseline or release-to-release upgrade migration. After a
|
||||||
|
release tag has shipped, released migration revision IDs are immutable.
|
||||||
|
|
||||||
|
The release policy is:
|
||||||
|
|
||||||
|
- unreleased migrations may be folded before release;
|
||||||
|
- released migrations are never rewritten or deleted;
|
||||||
|
- each stable release records the public migration head revisions in
|
||||||
|
`docs/migration-release-baselines.json`;
|
||||||
|
- fresh installations should apply release-level baselines/upgrades, not
|
||||||
|
unreleased create-then-rename churn;
|
||||||
|
- release-to-release schema changes should be folded into one reviewed
|
||||||
|
migration per migration owner where practical.
|
||||||
|
|
||||||
|
Audit the current graph during release preparation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Generate the reviewed/manual squash checklist:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --squash-plan
|
||||||
|
```
|
||||||
|
|
||||||
|
After the release migrations have been reviewed and the graph is final, record
|
||||||
|
the release baseline:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --record-release <x.y.z>
|
||||||
|
```
|
||||||
|
|
||||||
|
Use strict mode to verify that the current heads are recorded:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./.venv/bin/python scripts/release-migration-audit.py --strict
|
||||||
|
```
|
||||||
|
|
||||||
|
`scripts/push-release-tag.sh` runs the audit by default in automatic mode:
|
||||||
|
non-strict while no release baseline exists, strict after the first baseline is
|
||||||
|
recorded. Pass `--warn-migration-audit` for an explicit non-strict audit,
|
||||||
|
`--strict-migration-audit` to force strict mode, or `--skip-migration-audit`
|
||||||
|
only for emergency/manual release work.
|
||||||
|
|
||||||
|
Before the first stable release, fold the current development chain into the
|
||||||
|
first public baseline and record that baseline in
|
||||||
|
`docs/migration-release-baselines.json`. The tracking issue is
|
||||||
|
`add-ideas/govoplan-core#223`.
|
||||||
|
|
||||||
|
## Related Operator Documents
|
||||||
|
|
||||||
|
- `DEPLOYMENT_OPERATOR_GUIDE.md`: runtime environment, explicit migrations,
|
||||||
|
backup/restore commands, module installer daemon/supervisor operation, and
|
||||||
|
rollback drills.
|
||||||
|
- `REMOTE_WEBUI_BUNDLES.md`: experimental browser-loaded module bundles for
|
||||||
|
controlled deployments; normal releases use package builds.
|
||||||
|
|
||||||
## Release Checklist
|
## Release Checklist
|
||||||
|
|
||||||
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
- Keep Python package versions, WebUI package versions, and git tags aligned.
|
||||||
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign, calendar, and scaffold module repositories together.
|
- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign,
|
||||||
- Update `requirements-release.txt` and `webui/package.release.json` when the release tag changes.
|
calendar, and scaffold module repositories together.
|
||||||
- Generate the committed full-product release lockfile from `package.release.json` with `scripts/generate-release-lock.sh`.
|
- Update `requirements-release.txt` and `webui/package.release.json` when the
|
||||||
- Add separate release manifest/lockfile pairs only for module compositions that are shipped as their own products.
|
release tag changes.
|
||||||
|
- Generate the committed full-product release lockfile from
|
||||||
|
`package.release.json` with `scripts/generate-release-lock.sh`.
|
||||||
|
- Run `scripts/release-migration-audit.py --strict` after recording a release
|
||||||
|
baseline.
|
||||||
|
- Run the PostgreSQL release check against a disposable database.
|
||||||
|
- Publish the signed catalog through the release catalog publishing flow above.
|
||||||
|
- Add separate release manifest/lockfile pairs only for module compositions
|
||||||
|
that are shipped as their own products.
|
||||||
- Do not commit local sibling paths into release manifests.
|
- Do not commit local sibling paths into release manifests.
|
||||||
|
|||||||
161
docs/REMOTE_WEBUI_BUNDLES.md
Normal file
161
docs/REMOTE_WEBUI_BUNDLES.md
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
# Remote WebUI Bundle Loading
|
||||||
|
|
||||||
|
GovOPlaN WebUI modules normally ship through the core WebUI package graph:
|
||||||
|
install a tagged npm/git dependency, run `npm install`, rebuild the shell, and
|
||||||
|
restart or reload the served assets. Remote WebUI bundles are an experimental
|
||||||
|
future path for controlled deployments where a backend-enabled module is not in
|
||||||
|
the local WebUI package graph and the operator wants the shell to load its
|
||||||
|
frontend without rebuilding core.
|
||||||
|
|
||||||
|
This design defines the target guardrails. The current rebuild/reload path
|
||||||
|
remains the default production path.
|
||||||
|
|
||||||
|
## Current Rebuild Path
|
||||||
|
|
||||||
|
The supported release path is:
|
||||||
|
|
||||||
|
1. Install or remove backend and WebUI package dependencies through the trusted
|
||||||
|
installer CLI/daemon.
|
||||||
|
2. Snapshot package state before mutation.
|
||||||
|
3. Run `npm install` and optionally `npm run build`.
|
||||||
|
4. Restart or reload the served WebUI assets.
|
||||||
|
5. Use installer rollback if package apply, restart, or health checks fail.
|
||||||
|
|
||||||
|
Strengths:
|
||||||
|
|
||||||
|
- package manager and lockfile semantics stay conventional
|
||||||
|
- CSP can stay strict because all code is served as built assets
|
||||||
|
- rollback restores package files and lockfiles
|
||||||
|
- local development uses sibling workspace dependencies naturally
|
||||||
|
|
||||||
|
Costs:
|
||||||
|
|
||||||
|
- frontend changes require a rebuild/reload
|
||||||
|
- hot enabling a module with frontend code is not possible in the running shell
|
||||||
|
- failed rebuilds happen at install time, not at lazy module-load time
|
||||||
|
|
||||||
|
## Remote Bundle Path
|
||||||
|
|
||||||
|
Remote loading is only for modules that are enabled by the backend and absent
|
||||||
|
from `virtual:govoplan-installed-modules`. The backend manifest exposes:
|
||||||
|
|
||||||
|
- `FrontendModule.asset_manifest`
|
||||||
|
- `asset_manifest_integrity`
|
||||||
|
- `asset_manifest_signature`
|
||||||
|
- `asset_manifest_public_key_id`
|
||||||
|
- `asset_manifest_contract_version`
|
||||||
|
|
||||||
|
The WebUI shell fetches the asset manifest, verifies manifest integrity and/or
|
||||||
|
signature, validates the manifest contract, fetches the entry bundle, verifies
|
||||||
|
the entry integrity, imports the bundle, validates the exported
|
||||||
|
`PlatformWebModule`, and applies backend metadata before registering routes,
|
||||||
|
navigation, and UI capabilities.
|
||||||
|
|
||||||
|
Unsigned and unhashed manifests are skipped. Entries without integrity are
|
||||||
|
skipped. A module id mismatch is skipped.
|
||||||
|
|
||||||
|
## Asset Manifest Contract
|
||||||
|
|
||||||
|
Contract version `1`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"contractVersion": "1",
|
||||||
|
"moduleId": "files",
|
||||||
|
"entry": "./files-webui.remote.js",
|
||||||
|
"entryIntegrity": "SHA-256-<base64-or-hex-digest>",
|
||||||
|
"moduleExport": "default"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The backend manifest carries the manifest-level trust metadata. The remote
|
||||||
|
asset manifest carries the concrete entry URL and entry digest.
|
||||||
|
|
||||||
|
## Compatibility Checks
|
||||||
|
|
||||||
|
Before a remote bundle is accepted:
|
||||||
|
|
||||||
|
- backend module must be enabled
|
||||||
|
- local WebUI module with the same id must be absent
|
||||||
|
- manifest contract must be supported by the shell
|
||||||
|
- manifest `moduleId`, exported module `id`, and backend module id must match
|
||||||
|
- backend metadata remains authoritative for label, version, dependencies,
|
||||||
|
nav, and runtime UI capability exposure
|
||||||
|
- future contract versions must declare required shell capabilities so old
|
||||||
|
shells fail closed
|
||||||
|
|
||||||
|
Remote bundles must not broaden backend permissions. They can only contribute
|
||||||
|
routes/nav/capabilities that the backend metadata and existing permission
|
||||||
|
checks allow.
|
||||||
|
|
||||||
|
## CSP
|
||||||
|
|
||||||
|
The current implementation imports verified entry bytes through a blob URL. A
|
||||||
|
production CSP for this mode must explicitly allow:
|
||||||
|
|
||||||
|
- `connect-src` for the approved asset-manifest and bundle origins
|
||||||
|
- `script-src` for `blob:` only when remote loading is enabled
|
||||||
|
- no `unsafe-inline` requirement for remote modules
|
||||||
|
|
||||||
|
If an installation cannot allow `blob:` scripts, the follow-up implementation
|
||||||
|
should use signed same-origin module assets with static URLs instead of blob
|
||||||
|
imports. Remote loading must be disableable by configuration so strict-CSP
|
||||||
|
deployments can keep the rebuild path only.
|
||||||
|
|
||||||
|
## Cache Invalidation
|
||||||
|
|
||||||
|
The shell cache key is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<module-id>:<asset-manifest-url>:<backend-module-version>
|
||||||
|
```
|
||||||
|
|
||||||
|
Release catalogs should publish immutable manifest and entry URLs or change at
|
||||||
|
least one cache-key component on every release. Emergency rollback can point the
|
||||||
|
backend manifest at a previous immutable manifest URL or lower the enabled
|
||||||
|
module version after the backend package rollback.
|
||||||
|
|
||||||
|
Browsers may still cache remote responses. Approved asset servers should use:
|
||||||
|
|
||||||
|
- long cache lifetimes only for content-addressed immutable assets
|
||||||
|
- short cache lifetimes or explicit revalidation for channel/latest manifest
|
||||||
|
aliases
|
||||||
|
- `Cache-Control: no-store` for emergency override manifests
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Remote WebUI rollback is metadata rollback, not package-manager rollback:
|
||||||
|
|
||||||
|
- if the backend package install rolls back, the previous backend frontend
|
||||||
|
metadata returns
|
||||||
|
- if only remote assets are bad, publish a new manifest URL or revert the
|
||||||
|
backend/frontend metadata to the last known good manifest
|
||||||
|
- failed remote loading must degrade by omitting the remote module frontend,
|
||||||
|
not by breaking the shell
|
||||||
|
|
||||||
|
Installer run records should eventually include remote asset manifest URLs,
|
||||||
|
integrity, signature key ids, and load-test results when a plan enables a
|
||||||
|
remote frontend.
|
||||||
|
|
||||||
|
## Local And Development Behavior
|
||||||
|
|
||||||
|
Local development should keep using workspace/file dependencies and Vite. Remote
|
||||||
|
loading is useful for integration testing release artifacts, not for day-to-day
|
||||||
|
module UI development.
|
||||||
|
|
||||||
|
Development deployments may use unsigned catalogs and local asset servers only
|
||||||
|
when signature/integrity enforcement is intentionally disabled. The remote
|
||||||
|
loader itself still requires an integrity hash or a verifiable signature.
|
||||||
|
|
||||||
|
## Follow-Up Slices
|
||||||
|
|
||||||
|
1. Add a server-side remote-bundle policy flag and surface whether remote
|
||||||
|
loading is enabled in platform metadata.
|
||||||
|
2. Add CSP documentation/config generation for strict rebuild-only mode versus
|
||||||
|
controlled remote-bundle mode.
|
||||||
|
3. Add an installer/catalog preflight that validates remote WebUI asset
|
||||||
|
manifests and records verified identity in installer run records.
|
||||||
|
4. Add Playwright coverage for a signed test remote bundle, failed digest, bad
|
||||||
|
module id, cache-key refresh, and fallback when the module is unavailable.
|
||||||
|
5. Add release tooling to emit immutable remote asset manifests with digest,
|
||||||
|
signature, key id, and rollback metadata.
|
||||||
216
docs/UI_UX_DECISION_LEDGER.md
Normal file
216
docs/UI_UX_DECISION_LEDGER.md
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
# GovOPlaN UI/UX Decision Ledger
|
||||||
|
|
||||||
|
This ledger records product UI/UX decisions that affect admin, settings,
|
||||||
|
configuration, connector, policy, and module-management surfaces. It is a
|
||||||
|
binding design reference: future implementation should follow these decisions
|
||||||
|
unless the decision is explicitly revised here and affected screens are updated
|
||||||
|
to match.
|
||||||
|
|
||||||
|
Active tracking issue: `add-ideas/govoplan-core#225`.
|
||||||
|
|
||||||
|
## Operating Rule
|
||||||
|
|
||||||
|
GovOPlaN must expose advanced platform capability without presenting the user
|
||||||
|
with every option at once. Non-technical users should be able to complete common
|
||||||
|
workflows through guided, plain-language flows. Expert and diagnostic detail may
|
||||||
|
exist, but it must be deliberately layered.
|
||||||
|
|
||||||
|
The ethical design doctrine in `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` is the
|
||||||
|
normative baseline for this ledger. A screen can be visually quiet and still be
|
||||||
|
ethically complete only when it preserves context, consequence,
|
||||||
|
contestability, responsibility, and traceability at the point of action.
|
||||||
|
|
||||||
|
## Binding Decisions
|
||||||
|
|
||||||
|
| ID | Decision | Status | Applies To |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| UX-001 | Use progressive disclosure by default. Common decisions stay visible; advanced or hazardous options live in collapsed panels, later wizard steps, or explicit advanced sections. | Accepted | Admin, settings, connector setup, policy editors, module operations |
|
||||||
|
| UX-002 | Raw JSON is not a primary configuration editor. Every normal configuration path needs typed controls, validation, and help text. JSON may be shown for import/export, diagnostics, or expert inspection only. | Accepted | All admin/configuration UIs |
|
||||||
|
| UX-003 | Prefer guided workflows over option dumps for setup and risky changes. Use wizards for connector setup, configuration package import, module install/uninstall, destructive actions, and policy changes with broad impact. | Accepted | Connectors, package import, module lifecycle, governance/policy |
|
||||||
|
| UX-004 | Discover technical values when possible. Ask for the smallest user-known input, then discover and prefill technical fields for review. | Accepted | File connectors, mail/groupware, public URLs, future external providers |
|
||||||
|
| UX-005 | Disabled actions and failed steps must explain why they are unavailable, who can fix them, and where to go next. Silent disabled states are not acceptable for primary actions. | Accepted | All primary actions |
|
||||||
|
| UX-006 | Explanations must be available but quiet. Use short inline text and expose richer explanations through help affordances, side panels, expandable sections, or review steps. | Accepted | All complex forms and flows |
|
||||||
|
| UX-007 | Creation/editing should prefer modals or focused step flows when it reduces page clutter. Overview and comparison screens remain full-page. | Accepted | Settings/admin surfaces |
|
||||||
|
| UX-008 | Similar concepts must use shared placement and components: server/credential/policy rows, problem lists, review steps, advanced panels, confirmation modals, and empty/error states. | Accepted | Core WebUI and module WebUIs |
|
||||||
|
| UX-009 | Preflight and diagnostics are product UX. Validation, policy, permission, dependency, and capability failures must be written for operators before exposing internal details. | Accepted | Installer, connectors, policy, package import |
|
||||||
|
| UX-010 | Context, decision, and consequence must be visible together for actions that affect rights, duties, records, money, communication, retention, external systems, or workflow state. | Accepted | Workflow, admin, portal, policy, connector, records, payments |
|
||||||
|
| UX-011 | Navigation is not consent. Route changes, panel switches, and passive selection must not execute consequential actions without an explicit action surface. | Accepted | All WebUI surfaces |
|
||||||
|
| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit |
|
||||||
|
| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records |
|
||||||
|
| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments |
|
||||||
|
|
||||||
|
## Confirmed Implementation Decisions
|
||||||
|
|
||||||
|
These decisions were accepted on 2026-07-09 for the first implementation slice.
|
||||||
|
They shape reusable components and screen structure. Future changes must revise
|
||||||
|
this section and update affected screens.
|
||||||
|
|
||||||
|
### DUE-001: Primary Admin Configuration Shell
|
||||||
|
|
||||||
|
Decision: admin/configuration surfaces should standardize on a two-zone layout.
|
||||||
|
|
||||||
|
- Left or top area: searchable overview/list, status, and primary actions.
|
||||||
|
- Main area: selected item summary and common settings.
|
||||||
|
- Modal/wizard: create, connect, edit, test, review, and confirm actions.
|
||||||
|
- Collapsed advanced panels: rarely used technical fields.
|
||||||
|
|
||||||
|
Use this for file connectors and mail servers first, then migrate policy,
|
||||||
|
retention, API keys, and module operations.
|
||||||
|
|
||||||
|
### DUE-002: Wizard Step Model
|
||||||
|
|
||||||
|
Decision: standard wizard steps and names use this baseline:
|
||||||
|
|
||||||
|
1. Choose type or scope.
|
||||||
|
2. Enter essentials.
|
||||||
|
3. Discover or test.
|
||||||
|
4. Configure ownership and policy.
|
||||||
|
5. Review changes and blockers.
|
||||||
|
6. Save or submit for operator action.
|
||||||
|
|
||||||
|
Not every wizard needs every step, but flows should use these names and order
|
||||||
|
where applicable.
|
||||||
|
|
||||||
|
This applies to explicit assisted setup, onboarding, import, preflight, and
|
||||||
|
risky multi-step operations. It is not the default shape for ordinary create or
|
||||||
|
edit dialogs.
|
||||||
|
|
||||||
|
### DUE-003: Explanation Placement
|
||||||
|
|
||||||
|
Decision: richer explanations should use this placement model:
|
||||||
|
|
||||||
|
- Short helper text below labels only when it prevents common mistakes.
|
||||||
|
- Tooltips for icon-only controls and compact terms.
|
||||||
|
- Expandable "Why?" or "Details" blocks for contextual explanations.
|
||||||
|
- Right-side detail panel or review step for preflight/provenance/diagnostics.
|
||||||
|
|
||||||
|
Avoid permanently visible paragraphs inside dense admin cards.
|
||||||
|
|
||||||
|
### DUE-004: Advanced Options Contract
|
||||||
|
|
||||||
|
Decision: advanced options are fields that are needed for control, compatibility,
|
||||||
|
or diagnostics but are not part of the common setup path.
|
||||||
|
|
||||||
|
- protocol-specific endpoints, ports, path overrides, TLS/signing toggles,
|
||||||
|
timeout/retry tuning, raw headers, migration/destructive flags, and fallback
|
||||||
|
compatibility settings are advanced.
|
||||||
|
- names, descriptions, provider type, base URL, ownership, policy mode, and
|
||||||
|
basic credentials are not advanced.
|
||||||
|
|
||||||
|
Advanced fields must still be editable through typed controls.
|
||||||
|
|
||||||
|
### DUE-005: Blocker Language Contract
|
||||||
|
|
||||||
|
Decision: all disabled or blocked primary actions should use a shared structured
|
||||||
|
reason object.
|
||||||
|
|
||||||
|
Shape:
|
||||||
|
|
||||||
|
- `summary`: short plain-language reason.
|
||||||
|
- `details`: optional explanation.
|
||||||
|
- `required_action`: what needs to happen.
|
||||||
|
- `actor`: who can do it, for example system administrator or tenant admin.
|
||||||
|
- `target`: where to go or which setting/capability is missing.
|
||||||
|
- `technical_details`: optional expandable developer/operator data.
|
||||||
|
|
||||||
|
For simple missing required fields, highlight the fields and put the structured
|
||||||
|
reason in a compact hover/focus bubble on the disabled primary action instead of
|
||||||
|
adding a large persistent warning block.
|
||||||
|
|
||||||
|
### DUE-006: First Migration Surface
|
||||||
|
|
||||||
|
Decision: convert file connectors first, then mail servers. They share the same
|
||||||
|
server/credential/policy model, are high-value, and will prove the reusable
|
||||||
|
patterns quickly.
|
||||||
|
|
||||||
|
### DUE-007: Adaptive Create/Edit Forms
|
||||||
|
|
||||||
|
Decision: ordinary create/edit dialogs should show the full editable state in an
|
||||||
|
adaptive form, not force a linear wizard.
|
||||||
|
|
||||||
|
- The user chooses a type, provider, credential mode, or policy mode.
|
||||||
|
- The dialog immediately adjusts to show only the fields relevant to that state.
|
||||||
|
- Editing an existing object uses the same field grouping and layout as creating
|
||||||
|
it, so users can recognize the state they configured.
|
||||||
|
- Discovery and test actions must give visible feedback in the same dialog:
|
||||||
|
directly usable, discovered alternative, credentials needed/rejected, or not
|
||||||
|
found with the next action the user can take.
|
||||||
|
- Wizard shells remain available for assisted setup, first-run guidance,
|
||||||
|
imports, discovery-heavy flows, and operational preflight workflows.
|
||||||
|
|
||||||
|
## Implementation Sequence
|
||||||
|
|
||||||
|
| Phase | Scope | Output |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| 0 | UX inventory | List every admin/settings/configuration surface, classify it, and record whether it violates a binding decision. |
|
||||||
|
| 1 | Core primitives | Shared wizard shell, advanced panel, help affordance, blocker callout, problem list, review step, and discovery/test result components. |
|
||||||
|
| 2 | File connectors | Adaptive create/edit for provider/server, credential, discovery/test, ownership/policy, plus optional assisted wizard later. |
|
||||||
|
| 3 | Mail servers | Same adaptive pattern as files, adapted to server/credential/policy and test-send/test-login behavior. |
|
||||||
|
| 4 | Policy/retention editors | Effective value first, provenance visible, override/edit in modal, blocked edits explained. |
|
||||||
|
| 5 | Module/package operations | Step-based install/uninstall flow with preflight, maintenance, daemon handoff, migration, and rollback explanation. |
|
||||||
|
| 6 | Remaining settings/admin screens | Apply inventory findings by priority and remove one-off layouts. |
|
||||||
|
|
||||||
|
## Current Surface Inventory
|
||||||
|
|
||||||
|
This is the phase-0 inventory baseline. It should be extended as each screen is
|
||||||
|
converted or reviewed.
|
||||||
|
|
||||||
|
| Surface | Repository | UX State | Next Action |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| File connector settings | `govoplan-files` | First adaptive modal slice started: connections and credentials now use full-state create/edit forms with conditional fields, advanced panels, and blocker primitives. Wizard shell is retained for later assisted setup. Central policy card still needs a layered editor. | Finish provider discovery/test-in-flow, then convert policy editing. |
|
||||||
|
| Mail server settings | `govoplan-mail` / `govoplan-core` | Uses the shared server/credential model visually, but create/edit still needs the same adaptive pattern as files. | Migrate to adaptive server/credential/policy dialogs, with optional assisted wizard later. |
|
||||||
|
| Connector policy/effective rows | `govoplan-core`, module UIs | Effective-policy direction exists, but many editors still expose broad option sets. | Put effective value first, move overrides into modal, and explain blocked edits. |
|
||||||
|
| Admin module management | `govoplan-admin` | Has preflight concepts, but operational choices are still technical and dense. | Convert install/uninstall/package changes to operator wizards. |
|
||||||
|
| Configuration packages | `govoplan-admin` | Catalog/import work exists, but package editing can still drift toward technical fields. | Add guided import/review/problem-list flow. |
|
||||||
|
| Retention and privacy | `govoplan-core` | Functional editor exists; consequence language and provenance can be stronger. | Layer advanced retention options and add review for broad changes. |
|
||||||
|
| API keys | `govoplan-access` / admin UI | Security-sensitive creation needs least-privilege guidance. | Add scoped creation wizard with expiry/owner review. |
|
||||||
|
| User settings | `govoplan-core` | Preferences persistence exists; interface navigation issue was fixed earlier, but the surface still needs UX review. | Keep simple sections, remove double-click traps, and add quiet explanations. |
|
||||||
|
|
||||||
|
## Impact Index
|
||||||
|
|
||||||
|
| Surface | Current Risk | Expected Pattern |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| File connectors | Too many technical fields and unclear setup order. | Adaptive create/edit form with optional assisted wizard, discovery/test, credential binding, policy review, and advanced protocol panel. |
|
||||||
|
| Mail servers | Similar server/credential/policy concepts risk diverging from files. | Same tree/list and adaptive server/credential/policy model as file connectors. |
|
||||||
|
| Connector credentials | Security-sensitive details can overwhelm users. | Separate credential flow with secret-reference language and test result explanation. |
|
||||||
|
| Policy and effective settings | Users need to know why a value is inherited or locked. | Effective row first, source/provenance, local override action, actionable blocked reason. |
|
||||||
|
| Module install/uninstall | Operationally risky, currently inherently technical. | Operator wizard with preflight, maintenance, daemon handoff, review, and rollback explanation. |
|
||||||
|
| Configuration packages | Could become package JSON editing. | Package catalog/import wizard using provider data requirements and problem lists. |
|
||||||
|
| Retention/privacy | High-risk settings need explanation and provenance. | Layered editor with plain-language consequences and review. |
|
||||||
|
| Automation/workflow commands | Hidden side effects would undermine accountability. | Action/effect preview, system-actor display, command record, retry/quarantine/manual states, and audit links. |
|
||||||
|
| Postbox and encrypted communication | Retraction and access can be misunderstood. | Honest key-fetch/decryption state, expiry limits, recipient/device access provenance, and delivery evidence. |
|
||||||
|
| API keys | Security-sensitive creation and scope selection. | Scoped creation wizard, least-privilege suggestions, clear expiry/owner explanation. |
|
||||||
|
| User settings | Needs clarity and persistence across profile/interface/preferences. | Simple settings sections with immediate feedback and no double-click navigation traps. |
|
||||||
|
|
||||||
|
## Review Checklist
|
||||||
|
|
||||||
|
Every new or changed admin/configuration surface should answer:
|
||||||
|
|
||||||
|
- What is the common path, and is it visible without noise?
|
||||||
|
- Which fields are advanced, and are they collapsed by default?
|
||||||
|
- Is every configuration value editable through typed controls?
|
||||||
|
- Does the flow avoid JSON as the primary editor?
|
||||||
|
- Does the screen explain disabled actions and failed validation in plain
|
||||||
|
language?
|
||||||
|
- Does it say who can fix a blocker and where?
|
||||||
|
- Does it reuse existing core patterns for wizard steps, problem lists, modals,
|
||||||
|
help, and review?
|
||||||
|
- Is there a review or preflight step before broad, destructive, or risky
|
||||||
|
changes?
|
||||||
|
- Does the action surface show consequence, reversibility, and audit evidence
|
||||||
|
when rights, duties, records, money, communication, external systems, or
|
||||||
|
workflow state are affected?
|
||||||
|
- If automation is involved, can the user see the trigger, system actor,
|
||||||
|
observed effects, and failure/manual-intervention state?
|
||||||
|
- Are technical details available without being the first thing the user sees?
|
||||||
|
|
||||||
|
## Revision Rule
|
||||||
|
|
||||||
|
When a UX decision changes:
|
||||||
|
|
||||||
|
1. Update this ledger.
|
||||||
|
2. Update the affected shared components.
|
||||||
|
3. Update existing screens listed in the impact index.
|
||||||
|
4. Add or update Gitea issues for any remaining surfaces that still follow the
|
||||||
|
old decision.
|
||||||
|
5. Sync the wiki.
|
||||||
71
docs/audits/2026-07-09-dependency-audit.md
Normal file
71
docs/audits/2026-07-09-dependency-audit.md
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
# Dependency Audit - 2026-07-09
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-core
|
||||||
|
bash scripts/check-dependency-audits.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Status: remediated.
|
||||||
|
|
||||||
|
Initial result:
|
||||||
|
|
||||||
|
- Python audit failed: 24 advisories were reported across `cryptography`,
|
||||||
|
`pip`, `python-multipart`, `pyzipper`, and `starlette`.
|
||||||
|
- npm production audit passed: `npm audit --omit=dev` reported 0
|
||||||
|
vulnerabilities.
|
||||||
|
|
||||||
|
Python findings:
|
||||||
|
|
||||||
|
| Package | Installed | Advisory count | Minimum reported fix |
|
||||||
|
| --- | ---: | ---: | --- |
|
||||||
|
| `cryptography` | `44.0.0` | 5 | `48.0.1` |
|
||||||
|
| `pip` | `26.0.1` | 3 | `26.1.2` |
|
||||||
|
| `python-multipart` | `0.0.17` | 7 | `0.0.31` |
|
||||||
|
| `pyzipper` | `0.3.6` | 1 | `0.4.0` |
|
||||||
|
| `starlette` | `0.41.3` | 8 | `1.3.1` |
|
||||||
|
|
||||||
|
Private GovOPlaN packages were skipped by `pip-audit` because they are not
|
||||||
|
published on PyPI. That is expected for local editable development installs.
|
||||||
|
|
||||||
|
The remediation below upgrades those dependencies and records the compatibility
|
||||||
|
checks run against core, files, and campaign behavior.
|
||||||
|
|
||||||
|
## Remediation
|
||||||
|
|
||||||
|
Remediation applied on 2026-07-09:
|
||||||
|
|
||||||
|
- upgraded core's FastAPI floor to `fastapi>=0.139,<1`, resolving Starlette to
|
||||||
|
`starlette==1.3.1`
|
||||||
|
- upgraded core's cryptography floor to `cryptography>=48.0.1,<50`, resolving
|
||||||
|
to `cryptography==49.0.0`
|
||||||
|
- declared the files module upload parser dependency as
|
||||||
|
`python-multipart>=0.0.31,<1`, resolving to `python-multipart==0.0.32`
|
||||||
|
- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to
|
||||||
|
`pyzipper==0.4.0`
|
||||||
|
- upgraded the local audit environment to `pip==26.1.2`
|
||||||
|
- removed the obsolete local `govoplan-module-multimailer` editable install
|
||||||
|
from the audit environment so the audit reflects the split module product
|
||||||
|
|
||||||
|
Post-remediation result:
|
||||||
|
|
||||||
|
- `bash scripts/check-dependency-audits.sh`: passed, no known Python
|
||||||
|
vulnerabilities found and npm production audit reported 0 vulnerabilities.
|
||||||
|
- `python -m pip check`: passed.
|
||||||
|
- `bash scripts/check-module-matrix.sh`: passed.
|
||||||
|
- `python -m unittest tests.test_api_smoke`: passed.
|
||||||
|
- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- `pip-audit` still reports private GovOPlaN packages as skipped because they
|
||||||
|
are not published on PyPI. That is expected for editable local development
|
||||||
|
installs.
|
||||||
|
- `httpx2>=2.5,<3` is included in development requirements so Starlette's
|
||||||
|
`TestClient` uses the non-deprecated backend. `httpx==0.28.1` remains in dev
|
||||||
|
requirements for tests that mock connector HTTP responses directly.
|
||||||
|
- Split-module API routers now use Starlette's renamed
|
||||||
|
`HTTP_422_UNPROCESSABLE_CONTENT` status constant. This preserves the 422
|
||||||
|
status code while avoiding the deprecated `HTTP_422_UNPROCESSABLE_ENTITY`
|
||||||
|
alias.
|
||||||
@@ -179,6 +179,12 @@
|
|||||||
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
"description": "GovOPlaN Identity Trust module behavior or integration.",
|
||||||
"exclusive": false
|
"exclusive": false
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "module/identity",
|
||||||
|
"color": "bfd4f2",
|
||||||
|
"description": "GovOPlaN Identity module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "module/idm",
|
"name": "module/idm",
|
||||||
"color": "0052cc",
|
"color": "0052cc",
|
||||||
@@ -209,6 +215,12 @@
|
|||||||
"description": "GovOPlaN Ops module behavior or integration.",
|
"description": "GovOPlaN Ops module behavior or integration.",
|
||||||
"exclusive": false
|
"exclusive": false
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "module/organizations",
|
||||||
|
"color": "bfdadc",
|
||||||
|
"description": "GovOPlaN Organizations module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "module/payments",
|
"name": "module/payments",
|
||||||
"color": "bfd4f2",
|
"color": "bfd4f2",
|
||||||
@@ -227,6 +239,12 @@
|
|||||||
"description": "GovOPlaN Portal module behavior or integration.",
|
"description": "GovOPlaN Portal module behavior or integration.",
|
||||||
"exclusive": false
|
"exclusive": false
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "module/postbox",
|
||||||
|
"color": "d93f0b",
|
||||||
|
"description": "GovOPlaN Postbox module behavior or integration.",
|
||||||
|
"exclusive": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "module/reporting",
|
"name": "module/reporting",
|
||||||
"color": "c2e0c6",
|
"color": "c2e0c6",
|
||||||
|
|||||||
4
docs/migration-release-baselines.json
Normal file
4
docs/migration-release-baselines.json
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"version": 1,
|
||||||
|
"releases": []
|
||||||
|
}
|
||||||
@@ -15,6 +15,22 @@
|
|||||||
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
"python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||||
"webui_package": "@govoplan/files-webui",
|
"webui_package": "@govoplan/files-webui",
|
||||||
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-webui-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-webui-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
"license_features": ["module.files"],
|
"license_features": ["module.files"],
|
||||||
"tags": ["official", "service-module"]
|
"tags": ["official", "service-module"]
|
||||||
},
|
},
|
||||||
@@ -28,8 +44,53 @@
|
|||||||
"python_ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4",
|
"python_ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4",
|
||||||
"webui_package": "@govoplan/mail-webui",
|
"webui_package": "@govoplan/mail-webui",
|
||||||
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4",
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4",
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-webui-0.1.4.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-webui-0.1.4.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
"license_features": ["module.mail"],
|
"license_features": ["module.mail"],
|
||||||
"tags": ["official", "service-module"]
|
"tags": ["official", "service-module"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module_id": "dashboard",
|
||||||
|
"name": "Dashboard",
|
||||||
|
"description": "Configurable user home assembled from module-provided widgets.",
|
||||||
|
"version": "0.1.6",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-dashboard",
|
||||||
|
"python_ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6",
|
||||||
|
"webui_package": "@govoplan/dashboard-webui",
|
||||||
|
"webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6",
|
||||||
|
"artifact_integrity": {
|
||||||
|
"python": {
|
||||||
|
"ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6",
|
||||||
|
"sha256": "<sha256 of the resolved Python artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-0.1.6.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-0.1.6.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.6"
|
||||||
|
},
|
||||||
|
"webui": {
|
||||||
|
"ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6",
|
||||||
|
"sha256": "<sha256 of the resolved WebUI package artifact>",
|
||||||
|
"sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-webui-0.1.6.spdx.json",
|
||||||
|
"provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-webui-0.1.6.intoto.jsonl",
|
||||||
|
"git_ref": "refs/tags/v0.1.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"license_features": ["module.dashboard"],
|
||||||
|
"tags": ["official", "platform-module"]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"signatures": [
|
"signatures": [
|
||||||
|
|||||||
@@ -12,10 +12,10 @@ license = { file = "LICENSE" }
|
|||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"SQLAlchemy>=2.0,<3",
|
"SQLAlchemy>=2.0,<3",
|
||||||
"fastapi>=0.115,<1",
|
"fastapi>=0.139,<1",
|
||||||
"pydantic>=2,<3",
|
"pydantic>=2,<3",
|
||||||
"pydantic-settings>=2,<3",
|
"pydantic-settings>=2,<3",
|
||||||
"cryptography>=44,<45",
|
"cryptography>=48.0.1,<50",
|
||||||
"celery>=5,<6",
|
"celery>=5,<6",
|
||||||
"redis>=5,<6",
|
"redis>=5,<6",
|
||||||
"alembic>=1,<2",
|
"alembic>=1,<2",
|
||||||
@@ -34,8 +34,10 @@ govoplan-module-installer = "govoplan_core.commands.module_installer:main"
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
server = [
|
server = [
|
||||||
|
"psycopg[binary]>=3.2,<4",
|
||||||
"uvicorn[standard]>=0.32,<1",
|
"uvicorn[standard]>=0.32,<1",
|
||||||
]
|
]
|
||||||
dev = [
|
dev = [
|
||||||
"httpx==0.28.1",
|
"httpx==0.28.1",
|
||||||
|
"httpx2>=2.5,<3",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,11 +1,18 @@
|
|||||||
-e .[server]
|
-e .[server]
|
||||||
-e ../govoplan-tenancy
|
-e ../govoplan-tenancy
|
||||||
|
-e ../govoplan-organizations
|
||||||
|
-e ../govoplan-identity
|
||||||
-e ../govoplan-access
|
-e ../govoplan-access
|
||||||
-e ../govoplan-admin
|
-e ../govoplan-admin
|
||||||
-e ../govoplan-policy
|
-e ../govoplan-policy
|
||||||
-e ../govoplan-audit
|
-e ../govoplan-audit
|
||||||
|
-e ../govoplan-dashboard
|
||||||
-e ../govoplan-files
|
-e ../govoplan-files
|
||||||
-e ../govoplan-mail
|
-e ../govoplan-mail
|
||||||
-e ../govoplan-campaign
|
-e ../govoplan-campaign
|
||||||
-e ../govoplan-calendar
|
-e ../govoplan-calendar
|
||||||
|
-e ../govoplan-docs
|
||||||
|
-e ../govoplan-ops
|
||||||
httpx==0.28.1
|
httpx==0.28.1
|
||||||
|
httpx2>=2.5,<3
|
||||||
|
pip-audit>=2.9,<3
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
# Release install from tagged module repositories.
|
# Release install from tagged module repositories.
|
||||||
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
# Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release.
|
||||||
.[server]
|
.[server]
|
||||||
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.4
|
govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6
|
||||||
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.4
|
govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6
|
||||||
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.4
|
govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6
|
||||||
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.4
|
govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6
|
||||||
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.4
|
govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6
|
||||||
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4
|
govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6
|
||||||
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4
|
govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6
|
||||||
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.4
|
govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6
|
||||||
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.4
|
govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6
|
||||||
|
govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6
|
||||||
|
govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6
|
||||||
|
govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6
|
||||||
|
govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6
|
||||||
|
govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6
|
||||||
|
|||||||
37
scripts/check-dependency-audits.sh
Normal file
37
scripts/check-dependency-audits.sh
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||||
|
NODE_BIN="$(dirname "$NPM")"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
if [[ ! -x "$NPM" ]]; then
|
||||||
|
NPM=npm
|
||||||
|
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh"
|
||||||
|
|
||||||
|
if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then
|
||||||
|
echo "pip-audit is not installed. Install dev requirements first:" >&2
|
||||||
|
echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2
|
||||||
|
exit 127
|
||||||
|
fi
|
||||||
|
|
||||||
|
python_status=0
|
||||||
|
npm_status=0
|
||||||
|
|
||||||
|
"$PYTHON" -m pip_audit --progress-spinner off || python_status=$?
|
||||||
|
|
||||||
|
cd "$ROOT/webui"
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$?
|
||||||
|
|
||||||
|
if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then
|
||||||
|
echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
158
scripts/check-dependency-hygiene.sh
Normal file
158
scripts/check-dependency-hygiene.sh
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if importlib.util.find_spec("pip") is None:
|
||||||
|
print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr)
|
||||||
|
print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr)
|
||||||
|
print(" python -m venv .venv", file=sys.stderr)
|
||||||
|
print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr)
|
||||||
|
print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr)
|
||||||
|
raise SystemExit(127)
|
||||||
|
PY
|
||||||
|
|
||||||
|
"$PYTHON" -m pip check
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.metadata
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
prefix = pathlib.Path(sys.prefix)
|
||||||
|
legacy_patterns = (
|
||||||
|
"__editable__.govoplan_module_multimailer-*.pth",
|
||||||
|
"__editable___govoplan_module_multimailer_*_finder.py",
|
||||||
|
"govoplan_module_multimailer-*.dist-info",
|
||||||
|
)
|
||||||
|
problems: list[pathlib.Path] = []
|
||||||
|
|
||||||
|
for site_packages in (prefix / "lib").glob("python*/site-packages"):
|
||||||
|
for pattern in legacy_patterns:
|
||||||
|
problems.extend(site_packages.glob(pattern))
|
||||||
|
|
||||||
|
for dist in importlib.metadata.distributions():
|
||||||
|
if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer":
|
||||||
|
problems.append(pathlib.Path(str(dist.locate_file(""))))
|
||||||
|
|
||||||
|
if problems:
|
||||||
|
print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr)
|
||||||
|
for path in sorted(set(problems)):
|
||||||
|
print(f" {path}", file=sys.stderr)
|
||||||
|
print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
print("No stale legacy multimailer package metadata found.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
root = pathlib.Path.cwd()
|
||||||
|
scan_roots = [
|
||||||
|
root / "src",
|
||||||
|
root / "tests",
|
||||||
|
root.parent / "govoplan-access" / "src",
|
||||||
|
root.parent / "govoplan-admin" / "src",
|
||||||
|
root.parent / "govoplan-audit" / "src",
|
||||||
|
root.parent / "govoplan-calendar" / "src",
|
||||||
|
root.parent / "govoplan-campaign" / "src",
|
||||||
|
root.parent / "govoplan-dashboard" / "src",
|
||||||
|
root.parent / "govoplan-files" / "src",
|
||||||
|
root.parent / "govoplan-identity" / "src",
|
||||||
|
root.parent / "govoplan-mail" / "src",
|
||||||
|
root.parent / "govoplan-ops" / "src",
|
||||||
|
root.parent / "govoplan-organizations" / "src",
|
||||||
|
root.parent / "govoplan-policy" / "src",
|
||||||
|
root.parent / "govoplan-tenancy" / "src",
|
||||||
|
]
|
||||||
|
deprecated_tokens = {
|
||||||
|
"HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.",
|
||||||
|
}
|
||||||
|
|
||||||
|
hits: list[str] = []
|
||||||
|
for scan_root in scan_roots:
|
||||||
|
if not scan_root.exists():
|
||||||
|
continue
|
||||||
|
for path in scan_root.rglob("*.py"):
|
||||||
|
try:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
continue
|
||||||
|
for token, replacement in deprecated_tokens.items():
|
||||||
|
if token in text:
|
||||||
|
hits.append(f"{path}: deprecated {token}. {replacement}")
|
||||||
|
|
||||||
|
if hits:
|
||||||
|
print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr)
|
||||||
|
print("\n".join(hits), file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
print("Deprecated framework constant scan passed.")
|
||||||
|
PY
|
||||||
|
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=0
|
||||||
|
case "$CHECK_TESTCLIENT_DEPRECATIONS" in
|
||||||
|
0|false|False|no|No)
|
||||||
|
echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS"
|
||||||
|
;;
|
||||||
|
auto)
|
||||||
|
if "$PYTHON" - <<'PY'
|
||||||
|
import importlib.util
|
||||||
|
raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||||
|
else
|
||||||
|
echo "TestClient deprecation smoke skipped; install dev requirements to enable it."
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
RUN_TESTCLIENT_DEPRECATION_SMOKE=1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then
|
||||||
|
"$PYTHON" - <<'PY'
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
from starlette.exceptions import StarletteDeprecationWarning
|
||||||
|
|
||||||
|
warnings.simplefilter("error", StarletteDeprecationWarning)
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
|
||||||
|
@app.get("/dependency-hygiene-ping")
|
||||||
|
def ping() -> dict[str, bool]:
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
response = client.get("/dependency-hygiene-ping")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == {"ok": True}
|
||||||
|
|
||||||
|
print("TestClient deprecation smoke passed.")
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Dependency hygiene check passed."
|
||||||
@@ -5,11 +5,19 @@ ROOT="/mnt/DATA/git/govoplan-core"
|
|||||||
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin"
|
||||||
NPM="$NODE/npm"
|
NPM="$NODE/npm"
|
||||||
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
WEBUI_BIN="$ROOT/webui/node_modules/.bin"
|
||||||
|
NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")"
|
||||||
|
|
||||||
|
trap 'rm -f "$NPM_USERCONFIG"' EXIT
|
||||||
|
|
||||||
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
export PATH="$WEBUI_BIN:$NODE:$PATH"
|
||||||
|
export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG"
|
||||||
|
export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG"
|
||||||
|
unset npm_config_tmp NPM_CONFIG_TMP
|
||||||
|
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
|
CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh
|
||||||
|
|
||||||
./.venv/bin/python - <<'PY'
|
./.venv/bin/python - <<'PY'
|
||||||
import ast
|
import ast
|
||||||
import pathlib
|
import pathlib
|
||||||
@@ -20,6 +28,8 @@ roots = [
|
|||||||
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-access/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-admin/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"),
|
||||||
|
pathlib.Path("/mnt/DATA/git/govoplan-identity/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-policy/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-audit/src"),
|
||||||
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
pathlib.Path("/mnt/DATA/git/govoplan-mail/src"),
|
||||||
@@ -47,7 +57,7 @@ if errors:
|
|||||||
print(f"AST syntax check passed for {count} Python files")
|
print(f"AST syntax check passed for {count} Python files")
|
||||||
PY
|
PY
|
||||||
|
|
||||||
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")'
|
||||||
./.venv/bin/python scripts/check_dependency_boundaries.py
|
./.venv/bin/python scripts/check_dependency_boundaries.py
|
||||||
./.venv/bin/python -m unittest tests.test_module_system
|
./.venv/bin/python -m unittest tests.test_module_system
|
||||||
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
|
||||||
|
|||||||
23
scripts/check-module-matrix.sh
Normal file
23
scripts/check-module-matrix.sh
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}"
|
||||||
|
NODE_BIN="$(dirname "$NPM")"
|
||||||
|
|
||||||
|
if [[ ! -x "$PYTHON" ]]; then
|
||||||
|
PYTHON=python
|
||||||
|
fi
|
||||||
|
if [[ ! -x "$NPM" ]]; then
|
||||||
|
NPM=npm
|
||||||
|
NODE_BIN="$(dirname "$(command -v "$NPM")")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts
|
||||||
|
"$PYTHON" scripts/check_dependency_boundaries.py
|
||||||
|
|
||||||
|
cd "$ROOT/webui"
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities
|
||||||
|
PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations
|
||||||
@@ -11,8 +11,11 @@ REPOS = {
|
|||||||
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
"access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access",
|
||||||
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
"admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin",
|
||||||
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
"tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy",
|
||||||
|
"organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations",
|
||||||
|
"identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity",
|
||||||
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
"policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy",
|
||||||
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
"audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit",
|
||||||
|
"dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard",
|
||||||
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
"files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files",
|
||||||
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
"mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail",
|
||||||
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
"campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign",
|
||||||
@@ -22,15 +25,18 @@ PREFIXES = {
|
|||||||
"access": "govoplan_access",
|
"access": "govoplan_access",
|
||||||
"admin": "govoplan_admin",
|
"admin": "govoplan_admin",
|
||||||
"tenancy": "govoplan_tenancy",
|
"tenancy": "govoplan_tenancy",
|
||||||
|
"organizations": "govoplan_organizations",
|
||||||
|
"identity": "govoplan_identity",
|
||||||
"policy": "govoplan_policy",
|
"policy": "govoplan_policy",
|
||||||
"audit": "govoplan_audit",
|
"audit": "govoplan_audit",
|
||||||
|
"dashboard": "govoplan_dashboard",
|
||||||
"files": "govoplan_files",
|
"files": "govoplan_files",
|
||||||
"mail": "govoplan_mail",
|
"mail": "govoplan_mail",
|
||||||
"campaign": "govoplan_campaign",
|
"campaign": "govoplan_campaign",
|
||||||
}
|
}
|
||||||
FEATURE_OWNERS = ("files", "mail", "campaign")
|
FEATURE_OWNERS = ("files", "mail", "campaign")
|
||||||
PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit")
|
PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard")
|
||||||
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",)
|
PUBLIC_ACCESS_IMPORTS = ("govoplan_access.auth",)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
@@ -38,6 +38,22 @@ CATALOG_MODULES = (
|
|||||||
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
description="Tenant registry, tenant settings, and tenant resolution platform module.",
|
||||||
tags=("official", "platform-module"),
|
tags=("official", "platform-module"),
|
||||||
),
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="organizations",
|
||||||
|
repo="govoplan-organizations",
|
||||||
|
python_package="govoplan-organizations",
|
||||||
|
name="Organizations",
|
||||||
|
description="Organization units, functions, and account-held function assignments.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="identity",
|
||||||
|
repo="govoplan-identity",
|
||||||
|
python_package="govoplan-identity",
|
||||||
|
name="Identity",
|
||||||
|
description="Canonical identities and links between identities and platform accounts.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
),
|
||||||
CatalogModule(
|
CatalogModule(
|
||||||
module_id="access",
|
module_id="access",
|
||||||
repo="govoplan-access",
|
repo="govoplan-access",
|
||||||
@@ -72,6 +88,15 @@ CATALOG_MODULES = (
|
|||||||
description="Audit-log storage and audit administration routes.",
|
description="Audit-log storage and audit administration routes.",
|
||||||
tags=("official", "platform-module"),
|
tags=("official", "platform-module"),
|
||||||
),
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="dashboard",
|
||||||
|
repo="govoplan-dashboard",
|
||||||
|
python_package="govoplan-dashboard",
|
||||||
|
name="Dashboard",
|
||||||
|
description="Configurable user home assembled from module-provided dashboard widgets.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/dashboard-webui",
|
||||||
|
),
|
||||||
CatalogModule(
|
CatalogModule(
|
||||||
module_id="files",
|
module_id="files",
|
||||||
repo="govoplan-files",
|
repo="govoplan-files",
|
||||||
@@ -108,6 +133,24 @@ CATALOG_MODULES = (
|
|||||||
tags=("official", "service-module"),
|
tags=("official", "service-module"),
|
||||||
webui_package="@govoplan/calendar-webui",
|
webui_package="@govoplan/calendar-webui",
|
||||||
),
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="docs",
|
||||||
|
repo="govoplan-docs",
|
||||||
|
python_package="govoplan-docs",
|
||||||
|
name="Docs",
|
||||||
|
description="Configured-system documentation and evidence-aware help surfaces.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/docs-webui",
|
||||||
|
),
|
||||||
|
CatalogModule(
|
||||||
|
module_id="ops",
|
||||||
|
repo="govoplan-ops",
|
||||||
|
python_package="govoplan-ops",
|
||||||
|
name="Ops",
|
||||||
|
description="Runtime health, deployment profile, worker split, and sizing visibility.",
|
||||||
|
tags=("official", "platform-module"),
|
||||||
|
webui_package="@govoplan/ops-webui",
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,11 +12,10 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md")
|
||||||
DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md")
|
|
||||||
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md")
|
||||||
|
|
||||||
REPO_ROOTS = {
|
REPO_ROOTS = {
|
||||||
@@ -62,7 +61,7 @@ def main() -> int:
|
|||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG)
|
||||||
parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN)
|
parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import")
|
||||||
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO)
|
||||||
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
parser.add_argument("--apply", action="store_true", help="create missing Gitea issues")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -128,7 +127,7 @@ def main() -> int:
|
|||||||
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]:
|
||||||
if args.product_backlog.exists():
|
if args.product_backlog.exists():
|
||||||
yield from parse_product_backlog(args.product_backlog)
|
yield from parse_product_backlog(args.product_backlog)
|
||||||
if args.access_plan.exists():
|
if args.access_plan and args.access_plan.exists():
|
||||||
yield from parse_access_plan(args.access_plan)
|
yield from parse_access_plan(args.access_plan)
|
||||||
if args.web_todo.exists():
|
if args.web_todo.exists():
|
||||||
yield from parse_simple_todo(args.web_todo)
|
yield from parse_simple_todo(args.web_todo)
|
||||||
@@ -539,8 +538,7 @@ def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]:
|
|||||||
root = repo_root(REPO_ROOTS[repo_key])
|
root = repo_root(REPO_ROOTS[repo_key])
|
||||||
target = infer_target(root)
|
target = infer_target(root)
|
||||||
client = GiteaClient(target, token)
|
client = GiteaClient(target, token)
|
||||||
labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels"))
|
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label}
|
|
||||||
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100)
|
||||||
fingerprints: set[str] = set()
|
fingerprints: set[str] = set()
|
||||||
titles: set[str] = set()
|
titles: set[str] = set()
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token
|
||||||
|
|
||||||
|
|
||||||
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
GIT_ROOT = pathlib.Path("/mnt/DATA/git")
|
||||||
@@ -246,9 +246,7 @@ def is_excluded_repo_file(path: pathlib.Path) -> bool:
|
|||||||
return True
|
return True
|
||||||
if text.endswith("/docs/GITEA_ISSUES.md"):
|
if text.endswith("/docs/GITEA_ISSUES.md"):
|
||||||
return True
|
return True
|
||||||
if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"):
|
if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"):
|
||||||
return True
|
|
||||||
if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"):
|
|
||||||
return True
|
return True
|
||||||
if text.endswith("/govoplan-web/TODO.md"):
|
if text.endswith("/govoplan-web/TODO.md"):
|
||||||
return True
|
return True
|
||||||
@@ -537,8 +535,7 @@ def grouped_counts(candidates: list[Candidate]) -> dict[str, int]:
|
|||||||
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState:
|
||||||
target = infer_target(repo.root)
|
target = infer_target(repo.root)
|
||||||
client = GiteaClient(target, token)
|
client = GiteaClient(target, token)
|
||||||
labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50)
|
label_ids = label_ids_by_name(client, target.owner, target.repo)
|
||||||
label_ids = {str(label["name"]): int(label["id"]) for label in labels}
|
|
||||||
if apply:
|
if apply:
|
||||||
for name, color, description in GENERIC_LABELS:
|
for name, color, description in GENERIC_LABELS:
|
||||||
if name in label_ids:
|
if name in label_ids:
|
||||||
|
|||||||
250
scripts/gitea-migrate-org-labels.py
Normal file
250
scripts/gitea-migrate-org-labels.py
Normal file
@@ -0,0 +1,250 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Move issue labels from repository-local labels to organization labels."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from gitea_common import (
|
||||||
|
GiteaClient,
|
||||||
|
GiteaError,
|
||||||
|
infer_target,
|
||||||
|
load_dotenv,
|
||||||
|
org_path,
|
||||||
|
repo_path,
|
||||||
|
repo_root,
|
||||||
|
require_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference")
|
||||||
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
|
parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env"))
|
||||||
|
parser.add_argument("--org", help="organization owner; defaults to inferred owner")
|
||||||
|
parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated")
|
||||||
|
parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex")
|
||||||
|
parser.add_argument(
|
||||||
|
"--issue-mode",
|
||||||
|
choices=("replace", "delete-add"),
|
||||||
|
default="replace",
|
||||||
|
help="replace labels in one request, or delete local ids before adding org ids",
|
||||||
|
)
|
||||||
|
parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration")
|
||||||
|
parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = repo_root(args.root)
|
||||||
|
load_dotenv(args.env_file or root / ".env")
|
||||||
|
target = infer_target(root, args.remote)
|
||||||
|
owner = args.org or target.owner
|
||||||
|
client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN"))
|
||||||
|
if client.token is None:
|
||||||
|
raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.")
|
||||||
|
|
||||||
|
repo_filter = re.compile(args.repo_regex)
|
||||||
|
org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100))
|
||||||
|
if not org_labels:
|
||||||
|
raise GiteaError(f"{owner} has no organization labels")
|
||||||
|
|
||||||
|
repos = _selected_repositories(client, owner, args.repo, repo_filter)
|
||||||
|
totals = Totals()
|
||||||
|
print(f"Organization: {owner}")
|
||||||
|
print(f"Repositories: {len(repos)}")
|
||||||
|
print(f"Mode: {'apply' if args.apply else 'dry-run'}")
|
||||||
|
print(f"Delete repository labels: {args.delete_repo_labels}")
|
||||||
|
|
||||||
|
for index, repo in enumerate(repos, start=1):
|
||||||
|
repo_result = migrate_repository(
|
||||||
|
client,
|
||||||
|
owner=owner,
|
||||||
|
repo=repo,
|
||||||
|
org_labels=org_labels,
|
||||||
|
issue_mode=args.issue_mode,
|
||||||
|
apply=args.apply,
|
||||||
|
delete_repo_labels=args.delete_repo_labels,
|
||||||
|
)
|
||||||
|
totals.add(repo_result)
|
||||||
|
if repo_result.has_work:
|
||||||
|
print(
|
||||||
|
f"[{index}/{len(repos)}] {repo}: "
|
||||||
|
f"issue-labels={repo_result.issue_label_migrations}, "
|
||||||
|
f"delete-labels={repo_result.repo_label_deletions}, "
|
||||||
|
f"errors={len(repo_result.errors)}"
|
||||||
|
)
|
||||||
|
for error in repo_result.errors:
|
||||||
|
print(f" error: {error}")
|
||||||
|
else:
|
||||||
|
print(f"[{index}/{len(repos)}] {repo}: no matching local labels")
|
||||||
|
|
||||||
|
print("Summary:")
|
||||||
|
print(f" repositories processed: {totals.repositories}")
|
||||||
|
print(f" repositories with matching local labels: {totals.repositories_with_work}")
|
||||||
|
print(f" issue/PR local labels migrated: {totals.issue_label_migrations}")
|
||||||
|
print(f" repository labels deleted: {totals.repo_label_deletions}")
|
||||||
|
print(f" errors: {totals.errors}")
|
||||||
|
if totals.errors:
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
except GiteaError as exc:
|
||||||
|
print(f"error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
class RepoResult:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.issue_label_migrations = 0
|
||||||
|
self.repo_label_deletions = 0
|
||||||
|
self.errors: list[str] = []
|
||||||
|
self.matched_local_labels = 0
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_work(self) -> bool:
|
||||||
|
return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors)
|
||||||
|
|
||||||
|
|
||||||
|
class Totals:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.repositories = 0
|
||||||
|
self.repositories_with_work = 0
|
||||||
|
self.issue_label_migrations = 0
|
||||||
|
self.repo_label_deletions = 0
|
||||||
|
self.errors = 0
|
||||||
|
|
||||||
|
def add(self, result: RepoResult) -> None:
|
||||||
|
self.repositories += 1
|
||||||
|
if result.has_work:
|
||||||
|
self.repositories_with_work += 1
|
||||||
|
self.issue_label_migrations += result.issue_label_migrations
|
||||||
|
self.repo_label_deletions += result.repo_label_deletions
|
||||||
|
self.errors += len(result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_repository(
|
||||||
|
client: GiteaClient,
|
||||||
|
*,
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
org_labels: dict[str, dict[str, Any]],
|
||||||
|
issue_mode: str,
|
||||||
|
apply: bool,
|
||||||
|
delete_repo_labels: bool,
|
||||||
|
) -> RepoResult:
|
||||||
|
result = RepoResult()
|
||||||
|
repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100)
|
||||||
|
local_labels = {
|
||||||
|
str(label.get("name") or ""): label
|
||||||
|
for label in repo_labels
|
||||||
|
if str(label.get("name") or "") in org_labels
|
||||||
|
}
|
||||||
|
result.matched_local_labels = len(local_labels)
|
||||||
|
if not local_labels:
|
||||||
|
return result
|
||||||
|
|
||||||
|
local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None}
|
||||||
|
org_by_name = {name: _label_id(label) for name, label in org_labels.items()}
|
||||||
|
|
||||||
|
for issue_type in ("issues", "pulls"):
|
||||||
|
issues = client.paginate(
|
||||||
|
repo_path(owner, repo, "/issues"),
|
||||||
|
query={"state": "all", "type": issue_type},
|
||||||
|
limit=100,
|
||||||
|
)
|
||||||
|
for issue in issues:
|
||||||
|
issue_number = int(issue.get("number") or issue.get("index"))
|
||||||
|
issue_label_ids = [_label_id(label) for label in issue.get("labels") or []]
|
||||||
|
issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None]
|
||||||
|
final_label_ids = list(issue_label_ids)
|
||||||
|
changed = False
|
||||||
|
migrated_count = 0
|
||||||
|
local_ids_to_remove: list[int] = []
|
||||||
|
org_ids_to_add: list[int] = []
|
||||||
|
for label in issue.get("labels") or []:
|
||||||
|
local_id = _label_id(label)
|
||||||
|
if local_id is None or local_id not in local_by_id:
|
||||||
|
continue
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
org_id = org_by_name.get(name)
|
||||||
|
if org_id is None:
|
||||||
|
continue
|
||||||
|
local_ids_to_remove.append(local_id)
|
||||||
|
if org_id not in issue_label_ids and org_id not in org_ids_to_add:
|
||||||
|
org_ids_to_add.append(org_id)
|
||||||
|
final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id]
|
||||||
|
if org_id not in final_label_ids:
|
||||||
|
final_label_ids.append(org_id)
|
||||||
|
changed = True
|
||||||
|
migrated_count += 1
|
||||||
|
if changed:
|
||||||
|
if apply:
|
||||||
|
if issue_mode == "delete-add":
|
||||||
|
for local_id in local_ids_to_remove:
|
||||||
|
_remove_issue_label(client, owner, repo, issue_number, local_id)
|
||||||
|
if org_ids_to_add:
|
||||||
|
client.request_json(
|
||||||
|
"POST",
|
||||||
|
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||||
|
body={"labels": org_ids_to_add},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
client.request_json(
|
||||||
|
"PUT",
|
||||||
|
repo_path(owner, repo, f"/issues/{issue_number}/labels"),
|
||||||
|
body={"labels": final_label_ids},
|
||||||
|
)
|
||||||
|
result.issue_label_migrations += migrated_count
|
||||||
|
|
||||||
|
if delete_repo_labels:
|
||||||
|
for name, label in sorted(local_labels.items()):
|
||||||
|
label_id = _label_id(label)
|
||||||
|
if label_id is None:
|
||||||
|
result.errors.append(f"{name} has no label id")
|
||||||
|
continue
|
||||||
|
if apply:
|
||||||
|
try:
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}"))
|
||||||
|
except GiteaError as exc:
|
||||||
|
result.errors.append(f"delete repository label {name}: {exc}")
|
||||||
|
continue
|
||||||
|
result.repo_label_deletions += 1
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]:
|
||||||
|
if explicit:
|
||||||
|
return sorted(set(explicit))
|
||||||
|
repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100)
|
||||||
|
names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name"))
|
||||||
|
return [name for name in names if repo_filter.search(name)]
|
||||||
|
|
||||||
|
|
||||||
|
def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||||
|
return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None}
|
||||||
|
|
||||||
|
|
||||||
|
def _label_id(label: dict[str, Any]) -> int | None:
|
||||||
|
value = label.get("id") or label.get("index")
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return int(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None:
|
||||||
|
try:
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}"))
|
||||||
|
except GiteaError as exc:
|
||||||
|
if "HTTP 404" in str(exc):
|
||||||
|
return
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -15,6 +15,7 @@ from gitea_common import (
|
|||||||
infer_target,
|
infer_target,
|
||||||
load_dotenv,
|
load_dotenv,
|
||||||
load_json,
|
load_json,
|
||||||
|
org_path,
|
||||||
repo_path,
|
repo_path,
|
||||||
repo_root,
|
repo_root,
|
||||||
require_token,
|
require_token,
|
||||||
@@ -30,6 +31,13 @@ def main() -> int:
|
|||||||
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
parser.add_argument("--remote", default="origin", help="git remote to use for target inference")
|
||||||
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE)
|
||||||
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values")
|
||||||
|
parser.add_argument(
|
||||||
|
"--scope",
|
||||||
|
choices=("repository", "organization"),
|
||||||
|
default="repository",
|
||||||
|
help="sync repository labels or organization labels",
|
||||||
|
)
|
||||||
|
parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner")
|
||||||
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@@ -39,8 +47,12 @@ def main() -> int:
|
|||||||
target = infer_target(root, args.remote)
|
target = infer_target(root, args.remote)
|
||||||
labels = _load_labels(args.labels_file)
|
labels = _load_labels(args.labels_file)
|
||||||
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
token = require_token() if args.apply else os.environ.get("GITEA_TOKEN")
|
||||||
|
org_name = args.org or target.owner
|
||||||
|
|
||||||
print(f"Target: {target.display}")
|
if args.scope == "organization":
|
||||||
|
print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels")
|
||||||
|
else:
|
||||||
|
print(f"Target: {target.display}")
|
||||||
print(f"Labels file: {args.labels_file}")
|
print(f"Labels file: {args.labels_file}")
|
||||||
|
|
||||||
if not args.apply and not token:
|
if not args.apply and not token:
|
||||||
@@ -50,7 +62,10 @@ def main() -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
client = GiteaClient(target, token)
|
client = GiteaClient(target, token)
|
||||||
existing = _labels_by_name(client, target.owner, target.repo)
|
if args.scope == "organization":
|
||||||
|
existing = _org_labels_by_name(client, org_name)
|
||||||
|
else:
|
||||||
|
existing = _repo_labels_by_name(client, target.owner, target.repo)
|
||||||
creates: list[dict[str, Any]] = []
|
creates: list[dict[str, Any]] = []
|
||||||
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
updates: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||||
|
|
||||||
@@ -71,18 +86,17 @@ def main() -> int:
|
|||||||
for label in creates:
|
for label in creates:
|
||||||
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
print(f"{'create' if args.apply else 'would create'} {label['name']}")
|
||||||
if args.apply:
|
if args.apply:
|
||||||
client.request_json(
|
client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label)
|
||||||
"POST",
|
|
||||||
repo_path(target.owner, target.repo, "/labels"),
|
|
||||||
body=label,
|
|
||||||
)
|
|
||||||
|
|
||||||
for current, patch in updates:
|
for current, patch in updates:
|
||||||
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}")
|
||||||
if args.apply:
|
if args.apply:
|
||||||
|
label_id = current.get("id") or current.get("index")
|
||||||
|
if label_id is None:
|
||||||
|
raise GiteaError(f"{current['name']} has no label id in API response")
|
||||||
client.request_json(
|
client.request_json(
|
||||||
"PATCH",
|
"PATCH",
|
||||||
repo_path(target.owner, target.repo, f"/labels/{current['id']}"),
|
_update_path(args.scope, org_name, target.owner, target.repo, int(label_id)),
|
||||||
body=patch,
|
body=patch,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -119,11 +133,28 @@ def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]:
|
|||||||
return labels
|
return labels
|
||||||
|
|
||||||
|
|
||||||
def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]:
|
||||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
||||||
return {str(label.get("name")): label for label in labels}
|
return {str(label.get("name")): label for label in labels}
|
||||||
|
|
||||||
|
|
||||||
|
def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]:
|
||||||
|
labels = client.paginate(org_path(owner, "/labels"))
|
||||||
|
return {str(label.get("name")): label for label in labels}
|
||||||
|
|
||||||
|
|
||||||
|
def _create_path(scope: str, org: str, owner: str, repo: str) -> str:
|
||||||
|
if scope == "organization":
|
||||||
|
return org_path(org, "/labels")
|
||||||
|
return repo_path(owner, repo, "/labels")
|
||||||
|
|
||||||
|
|
||||||
|
def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str:
|
||||||
|
if scope == "organization":
|
||||||
|
return org_path(org, f"/labels/{label_id}")
|
||||||
|
return repo_path(owner, repo, f"/labels/{label_id}")
|
||||||
|
|
||||||
|
|
||||||
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]:
|
||||||
patch: dict[str, Any] = {}
|
patch: dict[str, Any] = {}
|
||||||
if _normalize_color(current.get("color")) != desired["color"]:
|
if _normalize_color(current.get("color")) != desired["color"]:
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ def main() -> int:
|
|||||||
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API")
|
||||||
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts")
|
||||||
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport")
|
||||||
|
parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered")
|
||||||
parser.add_argument("--apply", action="store_true")
|
parser.add_argument("--apply", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@@ -116,9 +117,17 @@ def main() -> int:
|
|||||||
overwrite_unmanaged=args.overwrite_unmanaged,
|
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||||
commit_message=args.commit_message,
|
commit_message=args.commit_message,
|
||||||
write_index=not bool(args.page),
|
write_index=not bool(args.page),
|
||||||
|
prune_managed=args.prune_managed and not bool(args.page),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
sync_repo_wiki(repo, repo_sources, token, overwrite_unmanaged=args.overwrite_unmanaged, write_index=not bool(args.page))
|
sync_repo_wiki(
|
||||||
|
repo,
|
||||||
|
repo_sources,
|
||||||
|
token,
|
||||||
|
overwrite_unmanaged=args.overwrite_unmanaged,
|
||||||
|
write_index=not bool(args.page),
|
||||||
|
prune_managed=args.prune_managed and not bool(args.page),
|
||||||
|
)
|
||||||
except GiteaError as exc:
|
except GiteaError as exc:
|
||||||
failures += 1
|
failures += 1
|
||||||
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr)
|
||||||
@@ -267,7 +276,15 @@ def preview_sources(sources: list[WikiSource]) -> None:
|
|||||||
print(f" ... {len(repo_sources) - 80} more")
|
print(f" ... {len(repo_sources) - 80} more")
|
||||||
|
|
||||||
|
|
||||||
def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, overwrite_unmanaged: bool, write_index: bool = True) -> None:
|
def sync_repo_wiki(
|
||||||
|
repo: RepoInfo,
|
||||||
|
sources: list[WikiSource],
|
||||||
|
token: str,
|
||||||
|
*,
|
||||||
|
overwrite_unmanaged: bool,
|
||||||
|
write_index: bool = True,
|
||||||
|
prune_managed: bool = False,
|
||||||
|
) -> None:
|
||||||
target = infer_target(repo.root)
|
target = infer_target(repo.root)
|
||||||
client = GiteaClient(target, token)
|
client = GiteaClient(target, token)
|
||||||
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
existing_pages = list_existing_wiki_pages(client, target.owner, target.repo)
|
||||||
@@ -311,6 +328,8 @@ def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, ove
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||||
|
if prune_managed and write_index:
|
||||||
|
prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources)
|
||||||
|
|
||||||
|
|
||||||
def sync_repo_wiki_git(
|
def sync_repo_wiki_git(
|
||||||
@@ -321,6 +340,7 @@ def sync_repo_wiki_git(
|
|||||||
overwrite_unmanaged: bool,
|
overwrite_unmanaged: bool,
|
||||||
commit_message: str,
|
commit_message: str,
|
||||||
write_index: bool = True,
|
write_index: bool = True,
|
||||||
|
prune_managed: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
target = infer_target(repo.root)
|
target = infer_target(repo.root)
|
||||||
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir)
|
||||||
@@ -353,6 +373,8 @@ def sync_repo_wiki_git(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync")
|
||||||
|
if prune_managed and write_index:
|
||||||
|
prune_managed_wiki_pages_git(wiki_root, sources)
|
||||||
if not git_has_changes(wiki_root):
|
if not git_has_changes(wiki_root):
|
||||||
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
print(f"unchanged {target.owner}/{target.repo} wiki repository")
|
||||||
return
|
return
|
||||||
@@ -426,6 +448,38 @@ def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, o
|
|||||||
return "updated"
|
return "updated"
|
||||||
|
|
||||||
|
|
||||||
|
def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None:
|
||||||
|
desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources}
|
||||||
|
desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md")
|
||||||
|
for path in sorted(wiki_root.glob("*.md")):
|
||||||
|
if path.name in desired:
|
||||||
|
continue
|
||||||
|
current = read_text(path)
|
||||||
|
if MANAGED_MARKER not in current:
|
||||||
|
continue
|
||||||
|
path.unlink()
|
||||||
|
print(f"deleted stale managed wiki page {path.name}")
|
||||||
|
|
||||||
|
|
||||||
|
def prune_managed_wiki_pages_api(
|
||||||
|
client: GiteaClient,
|
||||||
|
owner: str,
|
||||||
|
repo: str,
|
||||||
|
existing_page_names: dict[str, str],
|
||||||
|
sources: list[WikiSource],
|
||||||
|
) -> None:
|
||||||
|
desired = {source.page_title for source in sources}
|
||||||
|
desired.add("Codex-Project-Index")
|
||||||
|
for title, page_name in sorted(existing_page_names.items()):
|
||||||
|
if title in desired:
|
||||||
|
continue
|
||||||
|
current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||||
|
if MANAGED_MARKER not in decode_wiki_content(current):
|
||||||
|
continue
|
||||||
|
client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"))
|
||||||
|
print(f"deleted stale managed wiki page {owner}/{repo}:{title}")
|
||||||
|
|
||||||
|
|
||||||
def wiki_file_stem(title: str) -> str:
|
def wiki_file_stem(title: str) -> str:
|
||||||
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
return re.sub(r"[/\\]+", "-", title).strip() or "Home"
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token
|
from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token
|
||||||
|
|
||||||
|
|
||||||
MARKER_RE = re.compile(
|
MARKER_RE = re.compile(
|
||||||
@@ -347,8 +347,7 @@ def truncate_title(title: str, limit: int = 180) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]:
|
||||||
labels = client.paginate(repo_path(owner, repo, "/labels"))
|
return label_ids_by_name(client, owner, repo)
|
||||||
return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label}
|
|
||||||
|
|
||||||
|
|
||||||
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]:
|
||||||
|
|||||||
@@ -197,6 +197,47 @@ def repo_path(owner: str, repo: str, suffix: str) -> str:
|
|||||||
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def org_path(owner: str, suffix: str) -> str:
|
||||||
|
return f"/orgs/{quote_path(owner)}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]:
|
||||||
|
"""Return org labels plus optional repository labels, keyed by label name.
|
||||||
|
|
||||||
|
Gitea stores organization labels separately from repository labels. Issue
|
||||||
|
APIs can use label ids from both scopes on supported instances, so issue
|
||||||
|
creation helpers should resolve both. Repository labels intentionally win
|
||||||
|
when a repo carries a local label with the same name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
labels: dict[str, dict[str, Any]] = {}
|
||||||
|
try:
|
||||||
|
for label in client.paginate(org_path(owner, "/labels")):
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
if name:
|
||||||
|
labels[name] = label
|
||||||
|
except GiteaError:
|
||||||
|
# User-owned repositories or older instances may not expose org labels.
|
||||||
|
pass
|
||||||
|
|
||||||
|
if repo:
|
||||||
|
for label in client.paginate(repo_path(owner, repo, "/labels")):
|
||||||
|
name = str(label.get("name") or "")
|
||||||
|
if name:
|
||||||
|
labels[name] = label
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]:
|
||||||
|
ids: dict[str, int] = {}
|
||||||
|
for name, label in labels_by_name(client, owner, repo).items():
|
||||||
|
label_id = label.get("id") or label.get("index")
|
||||||
|
if label_id is None:
|
||||||
|
continue
|
||||||
|
ids[name] = int(label_id)
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
def _git_remote(root: pathlib.Path, remote_name: str) -> str:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["git", "-C", str(root), "remote", "get-url", remote_name],
|
["git", "-C", str(root), "remote", "get-url", remote_name],
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}"
|
|||||||
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}"
|
||||||
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}"
|
||||||
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}"
|
||||||
|
DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}"
|
||||||
|
|
||||||
LOG_DIR="$ROOT/runtime/dev-launcher"
|
LOG_DIR="$ROOT/runtime/dev-launcher"
|
||||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||||
@@ -31,6 +32,20 @@ fail() {
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "$DEV_DATABASE_BACKEND" in
|
||||||
|
postgres)
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}"
|
||||||
|
;;
|
||||||
|
sqlite)
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||||
|
|
||||||
port_is_free() {
|
port_is_free() {
|
||||||
"$PYTHON" - "$1" "$2" <<'PY'
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
import socket
|
import socket
|
||||||
@@ -139,6 +154,7 @@ GovOPlaN is running.
|
|||||||
Web UI: $FRONTEND_URL
|
Web UI: $FRONTEND_URL
|
||||||
API: $BACKEND_URL/api/v1
|
API: $BACKEND_URL/api/v1
|
||||||
Health: $BACKEND_URL/health
|
Health: $BACKEND_URL/health
|
||||||
|
DB: $DATABASE_URL
|
||||||
|
|
||||||
Logs:
|
Logs:
|
||||||
Backend: $BACKEND_LOG
|
Backend: $BACKEND_LOG
|
||||||
|
|||||||
238
scripts/launch-production-like-dev.sh
Normal file
238
scripts/launch-production-like-dev.sh
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}"
|
||||||
|
PROFILE_ROOT="$ROOT/dev/production-like"
|
||||||
|
PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}"
|
||||||
|
if [ ! -f "$PROFILE_ENV" ]; then
|
||||||
|
PROFILE_ENV="$PROFILE_ROOT/.env.example"
|
||||||
|
fi
|
||||||
|
|
||||||
|
WEBUI_ROOT="$ROOT/webui"
|
||||||
|
PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
|
||||||
|
NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}"
|
||||||
|
NPM="${NPM:-$NODE_BIN/npm}"
|
||||||
|
DOCKER="${DOCKER:-docker}"
|
||||||
|
|
||||||
|
BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}"
|
||||||
|
BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}"
|
||||||
|
FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}"
|
||||||
|
FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}"
|
||||||
|
OPEN_BROWSER="${OPEN_BROWSER:-1}"
|
||||||
|
STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}"
|
||||||
|
|
||||||
|
LOG_DIR="$ROOT/runtime/production-like/logs"
|
||||||
|
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||||
|
WORKER_LOG="$LOG_DIR/worker.log"
|
||||||
|
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||||
|
BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT"
|
||||||
|
FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT"
|
||||||
|
|
||||||
|
backend_pid=""
|
||||||
|
worker_pid=""
|
||||||
|
frontend_pid=""
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
printf 'launch-production-like-dev: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV"
|
||||||
|
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$PROFILE_ENV"
|
||||||
|
set +a
|
||||||
|
|
||||||
|
export APP_ENV="${APP_ENV:-staging}"
|
||||||
|
export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops}"
|
||||||
|
export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
|
||||||
|
export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
|
||||||
|
export CELERY_ENABLED="${CELERY_ENABLED:-true}"
|
||||||
|
export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}"
|
||||||
|
export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
|
||||||
|
export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}"
|
||||||
|
export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
|
||||||
|
export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}"
|
||||||
|
export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}"
|
||||||
|
|
||||||
|
if [ -z "${MASTER_KEY_B64:-}" ]; then
|
||||||
|
MASTER_KEY_B64="$("$PYTHON" - <<'PY'
|
||||||
|
from cryptography.fernet import Fernet
|
||||||
|
print(Fernet.generate_key().decode())
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
export MASTER_KEY_B64
|
||||||
|
fi
|
||||||
|
|
||||||
|
compose() {
|
||||||
|
"$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
port_is_free() {
|
||||||
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
host = sys.argv[1]
|
||||||
|
port = int(sys.argv[2])
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
try:
|
||||||
|
sock.bind((host, port))
|
||||||
|
except OSError:
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_tcp() {
|
||||||
|
"$PYTHON" - "$1" "$2" <<'PY'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
host = sys.argv[1]
|
||||||
|
port = int(sys.argv[2])
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
last_error = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with socket.create_connection((host, port), timeout=2):
|
||||||
|
raise SystemExit(0)
|
||||||
|
except OSError as exc:
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_url() {
|
||||||
|
"$PYTHON" - "$1" <<'PY'
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
url = sys.argv[1]
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
last_error = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=2) as response:
|
||||||
|
if 200 <= response.status < 500:
|
||||||
|
raise SystemExit(0)
|
||||||
|
except Exception as exc: # noqa: BLE001 - printed only on timeout.
|
||||||
|
last_error = exc
|
||||||
|
time.sleep(1)
|
||||||
|
print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then
|
||||||
|
kill "$frontend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then
|
||||||
|
kill "$backend_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then
|
||||||
|
kill "$worker_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then
|
||||||
|
compose down >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt"
|
||||||
|
[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation."
|
||||||
|
[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json"
|
||||||
|
[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install"
|
||||||
|
|
||||||
|
mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT"
|
||||||
|
: > "$BACKEND_LOG"
|
||||||
|
: > "$WORKER_LOG"
|
||||||
|
: > "$FRONTEND_LOG"
|
||||||
|
|
||||||
|
port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use"
|
||||||
|
port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use"
|
||||||
|
|
||||||
|
printf 'Starting production-like dependencies through Docker Compose\n'
|
||||||
|
compose up -d
|
||||||
|
|
||||||
|
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}"
|
||||||
|
wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}"
|
||||||
|
|
||||||
|
printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data
|
||||||
|
)
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT"
|
||||||
|
) >"$BACKEND_LOG" 2>&1 &
|
||||||
|
backend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s/health\n' "$BACKEND_URL"
|
||||||
|
wait_for_url "$BACKEND_URL/health" || {
|
||||||
|
tail -n 80 "$BACKEND_LOG" >&2 || true
|
||||||
|
fail "backend did not become healthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES"
|
||||||
|
(
|
||||||
|
cd "$ROOT"
|
||||||
|
"$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \
|
||||||
|
--queues "$CELERY_QUEUES" \
|
||||||
|
--hostname "govoplan-production-like@%h" \
|
||||||
|
--loglevel INFO
|
||||||
|
) >"$WORKER_LOG" 2>&1 &
|
||||||
|
worker_pid="$!"
|
||||||
|
|
||||||
|
printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL"
|
||||||
|
(
|
||||||
|
cd "$WEBUI_ROOT"
|
||||||
|
PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \
|
||||||
|
CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \
|
||||||
|
CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \
|
||||||
|
VITE_API_PROXY_TARGET="$BACKEND_URL" \
|
||||||
|
"$NPM" run dev -- --force
|
||||||
|
) >"$FRONTEND_LOG" 2>&1 &
|
||||||
|
frontend_pid="$!"
|
||||||
|
|
||||||
|
printf 'Waiting for %s\n' "$FRONTEND_URL"
|
||||||
|
wait_for_url "$FRONTEND_URL" || {
|
||||||
|
tail -n 80 "$FRONTEND_LOG" >&2 || true
|
||||||
|
fail "frontend did not become reachable"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then
|
||||||
|
xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat <<EOF
|
||||||
|
|
||||||
|
GovOPlaN production-like development profile is running.
|
||||||
|
Web UI: $FRONTEND_URL
|
||||||
|
API: $BACKEND_URL/api/v1
|
||||||
|
Health: $BACKEND_URL/health
|
||||||
|
Ops: $BACKEND_URL/api/v1/ops/status
|
||||||
|
DB: $DATABASE_URL
|
||||||
|
Redis: $REDIS_URL
|
||||||
|
Files: $FILE_STORAGE_LOCAL_ROOT
|
||||||
|
|
||||||
|
Logs:
|
||||||
|
Backend: $BACKEND_LOG
|
||||||
|
Worker: $WORKER_LOG
|
||||||
|
WebUI: $FRONTEND_LOG
|
||||||
|
|
||||||
|
Docker dependencies remain running after Ctrl+C unless GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1.
|
||||||
|
Press Ctrl+C to stop API, worker, and WebUI.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
wait
|
||||||
512
scripts/module-installer-rollback-drill.py
Normal file
512
scripts/module-installer-rollback-drill.py
Normal file
@@ -0,0 +1,512 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Callable, Iterable
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SRC = ROOT / "src"
|
||||||
|
if str(SRC) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SRC))
|
||||||
|
|
||||||
|
os.environ.setdefault("APP_ENV", "test")
|
||||||
|
os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false")
|
||||||
|
os.environ.setdefault("CELERY_ENABLED", "false")
|
||||||
|
|
||||||
|
from govoplan_core.admin.models import SystemSettings
|
||||||
|
from govoplan_core.core.maintenance import MaintenanceMode, save_maintenance_mode
|
||||||
|
from govoplan_core.core.module_installer import (
|
||||||
|
cancel_module_installer_request,
|
||||||
|
claim_next_module_installer_request,
|
||||||
|
module_installer_daemon_status,
|
||||||
|
module_installer_lock_status,
|
||||||
|
queue_module_installer_request,
|
||||||
|
read_module_installer_run,
|
||||||
|
retry_module_installer_request,
|
||||||
|
rollback_module_install_run,
|
||||||
|
run_module_install_plan,
|
||||||
|
supervise_module_install_plan,
|
||||||
|
update_module_installer_daemon_status,
|
||||||
|
update_module_installer_request,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.module_management import (
|
||||||
|
ModuleInstallPlan,
|
||||||
|
ModuleInstallPlanItem,
|
||||||
|
saved_desired_enabled_modules,
|
||||||
|
saved_module_install_plan,
|
||||||
|
save_desired_enabled_modules,
|
||||||
|
save_module_install_plan,
|
||||||
|
)
|
||||||
|
from govoplan_core.core.modules import MigrationRetirementPlan, MigrationSpec, ModuleManifest
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.db.session import configure_database, get_database
|
||||||
|
from govoplan_core.server.registry import available_module_manifests
|
||||||
|
|
||||||
|
|
||||||
|
Scenario = Callable[[Path], dict[str, object]]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.")
|
||||||
|
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.")
|
||||||
|
parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.")
|
||||||
|
parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.")
|
||||||
|
parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-"))
|
||||||
|
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
selected = args.scenario or sorted(SCENARIOS)
|
||||||
|
results: list[dict[str, object]] = []
|
||||||
|
ok = True
|
||||||
|
try:
|
||||||
|
for name in selected:
|
||||||
|
scenario_root = runtime_root / name
|
||||||
|
scenario_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
try:
|
||||||
|
result = SCENARIOS[name](scenario_root)
|
||||||
|
except Exception as exc:
|
||||||
|
ok = False
|
||||||
|
result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)}
|
||||||
|
else:
|
||||||
|
ok = ok and bool(result.get("ok"))
|
||||||
|
results.append(result)
|
||||||
|
finally:
|
||||||
|
if not args.keep_runtime and args.runtime_root is None:
|
||||||
|
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||||
|
|
||||||
|
payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results}
|
||||||
|
if args.format == "json":
|
||||||
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||||
|
else:
|
||||||
|
for result in results:
|
||||||
|
status = "ok" if result.get("ok") else "FAILED"
|
||||||
|
print(f"{status}: {result['scenario']}")
|
||||||
|
if result.get("run_id"):
|
||||||
|
print(f" run: {result['run_id']}")
|
||||||
|
if result.get("record_path"):
|
||||||
|
print(f" record: {result['record_path']}")
|
||||||
|
if result.get("error"):
|
||||||
|
print(f" error: {result['error']}")
|
||||||
|
print(f"runtime: {runtime_root}")
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
def package_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "package-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_package_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
)
|
||||||
|
restored_plan = saved_module_install_plan(session)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"package-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
"plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def migration_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "migration-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_migration_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
migrate_database=True,
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"migration-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def restart_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "restart-failure-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_restart_failure_run):
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
restart_command="restart-fails",
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"restart-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def health_timeout(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
plan = _saved_install_plan(database, "health-timeout-example")
|
||||||
|
with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen():
|
||||||
|
result = supervise_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
restart_command="restart-ok",
|
||||||
|
health_url="http://127.0.0.1:9/health",
|
||||||
|
health_timeout_seconds=0.2,
|
||||||
|
health_interval_seconds=0.05,
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"health-timeout",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")),
|
||||||
|
"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def destructive_retirement_failure(root: Path) -> dict[str, object]:
|
||||||
|
database_url, database = _sqlite_database(root)
|
||||||
|
available = {"retirement-example": _retirement_failure_manifest()}
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": "retirement-example",
|
||||||
|
"action": "uninstall",
|
||||||
|
"python_package": "govoplan-retirement-example",
|
||||||
|
"destroy_data": True,
|
||||||
|
}])
|
||||||
|
session.commit()
|
||||||
|
with _patch_subprocess(_success_run):
|
||||||
|
result = run_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available,
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url=database_url,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
)
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
return _scenario_result(
|
||||||
|
"destructive-retirement-failure",
|
||||||
|
root,
|
||||||
|
result,
|
||||||
|
expected_status="rolled-back",
|
||||||
|
checks={
|
||||||
|
"rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict),
|
||||||
|
"sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_hooks(root: Path) -> dict[str, object]:
|
||||||
|
_, database = _sqlite_database(root)
|
||||||
|
backup_script = (
|
||||||
|
"import json, os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
check_script = (
|
||||||
|
"import os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text("
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
restore_script = (
|
||||||
|
"import os, pathlib; "
|
||||||
|
"pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text("
|
||||||
|
"os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')"
|
||||||
|
)
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": "postgres-hook-example",
|
||||||
|
"action": "install",
|
||||||
|
"python_package": "govoplan-postgres-hook-example",
|
||||||
|
"python_ref": "govoplan-postgres-hook-example==0.1.0",
|
||||||
|
}])
|
||||||
|
session.commit()
|
||||||
|
result = run_module_install_plan(
|
||||||
|
session=session,
|
||||||
|
plan=plan,
|
||||||
|
available=available_module_manifests(),
|
||||||
|
current_enabled=("tenancy", "access"),
|
||||||
|
desired_enabled=("tenancy", "access"),
|
||||||
|
database_url="postgresql://db.example.invalid/govoplan",
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
migrate_database=True,
|
||||||
|
database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}",
|
||||||
|
database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}",
|
||||||
|
database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}",
|
||||||
|
dry_run=True,
|
||||||
|
)
|
||||||
|
with _patch_subprocess(_non_shell_success_run):
|
||||||
|
rollback = rollback_module_install_run(
|
||||||
|
run_id=result.run_id,
|
||||||
|
runtime_dir=root / "installer",
|
||||||
|
database_url="postgresql://db.example.invalid/govoplan",
|
||||||
|
)
|
||||||
|
run_dir = result.record_path.parent
|
||||||
|
record = _run_record(root, result.run_id)
|
||||||
|
ok = (
|
||||||
|
result.status == "dry-run"
|
||||||
|
and rollback.status == "rolled-back"
|
||||||
|
and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup"
|
||||||
|
and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan"
|
||||||
|
and isinstance(_nested(record, "snapshot", "database_backup"), dict)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"scenario": "postgres-hooks",
|
||||||
|
"ok": ok,
|
||||||
|
"root": str(root),
|
||||||
|
"run_id": result.run_id,
|
||||||
|
"record_path": str(result.record_path),
|
||||||
|
"rollback_status": rollback.status,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def queue_and_lock(root: Path) -> dict[str, object]:
|
||||||
|
runtime_dir = root / "installer"
|
||||||
|
status = update_module_installer_daemon_status(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
patch={"status": "polling", "current_request_id": None},
|
||||||
|
)
|
||||||
|
request = queue_module_installer_request(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
requested_by="drill",
|
||||||
|
options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]},
|
||||||
|
)
|
||||||
|
claimed = claim_next_module_installer_request(runtime_dir=runtime_dir)
|
||||||
|
assert claimed is not None
|
||||||
|
failed = update_module_installer_request(
|
||||||
|
runtime_dir=runtime_dir,
|
||||||
|
request_id=str(claimed["request_id"]),
|
||||||
|
patch={"status": "failed", "error": "simulated drill failure"},
|
||||||
|
)
|
||||||
|
retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill")
|
||||||
|
cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill")
|
||||||
|
lock_path = runtime_dir / "install.lock"
|
||||||
|
lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8")
|
||||||
|
lock = module_installer_lock_status(runtime_dir=runtime_dir)
|
||||||
|
lock_path.unlink()
|
||||||
|
return {
|
||||||
|
"scenario": "queue-and-lock",
|
||||||
|
"ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"],
|
||||||
|
"root": str(root),
|
||||||
|
"daemon_status_path": str(runtime_dir / "daemon.status.json"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_database(root: Path) -> tuple[str, Any]:
|
||||||
|
database_url = f"sqlite:///{root / 'drill.db'}"
|
||||||
|
configure_database(database_url)
|
||||||
|
database = get_database()
|
||||||
|
Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__])
|
||||||
|
return database_url, database
|
||||||
|
|
||||||
|
|
||||||
|
def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan:
|
||||||
|
with database.session() as session:
|
||||||
|
save_maintenance_mode(session, MaintenanceMode(enabled=True))
|
||||||
|
plan = save_module_install_plan(session, [{
|
||||||
|
"module_id": module_id,
|
||||||
|
"action": "install",
|
||||||
|
"python_package": f"govoplan-{module_id}",
|
||||||
|
"python_ref": f"govoplan-{module_id}==0.1.0",
|
||||||
|
}])
|
||||||
|
save_desired_enabled_modules(session, ("tenancy", "access"))
|
||||||
|
session.commit()
|
||||||
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
def _retirement_failure_manifest() -> ModuleManifest:
|
||||||
|
def destroy(_session: object, _module_id: str) -> None:
|
||||||
|
raise RuntimeError("simulated destructive retirement failure")
|
||||||
|
|
||||||
|
def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan:
|
||||||
|
return MigrationRetirementPlan(
|
||||||
|
supported=True,
|
||||||
|
summary="Drill retirement provider supports destructive cleanup.",
|
||||||
|
destroy_data_supported=True,
|
||||||
|
destroy_data_summary="Drill destructive cleanup will fail during execution.",
|
||||||
|
destroy_data_executor=destroy,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ModuleManifest(
|
||||||
|
id="retirement-example",
|
||||||
|
name="Retirement example",
|
||||||
|
version="0.1.0",
|
||||||
|
migration_spec=MigrationSpec(
|
||||||
|
module_id="retirement-example",
|
||||||
|
retirement_supported=True,
|
||||||
|
retirement_provider=provider,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _scenario_result(
|
||||||
|
scenario: str,
|
||||||
|
root: Path,
|
||||||
|
result: Any,
|
||||||
|
*,
|
||||||
|
expected_status: str,
|
||||||
|
checks: dict[str, bool],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"scenario": scenario,
|
||||||
|
"ok": result.status == expected_status and all(checks.values()),
|
||||||
|
"root": str(root),
|
||||||
|
"run_id": result.run_id,
|
||||||
|
"record_path": str(result.record_path),
|
||||||
|
"status": result.status,
|
||||||
|
"checks": checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _run_record(root: Path, run_id: str) -> dict[str, object]:
|
||||||
|
return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _nested(value: dict[str, object], *keys: str) -> object:
|
||||||
|
current: object = value
|
||||||
|
for key in keys:
|
||||||
|
if not isinstance(current, dict):
|
||||||
|
return None
|
||||||
|
current = current.get(key)
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
|
def _command_text(argv: object) -> str:
|
||||||
|
if isinstance(argv, list | tuple):
|
||||||
|
return " ".join(str(item) for item in argv)
|
||||||
|
return str(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "pip freeze" in text:
|
||||||
|
return _completed(stdout="govoplan-core==0.0.0\n")
|
||||||
|
return _completed()
|
||||||
|
|
||||||
|
|
||||||
|
def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any:
|
||||||
|
if kwargs.get("shell"):
|
||||||
|
return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs)
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "pip install" in text and "==" in text and "-r" not in text:
|
||||||
|
return _completed(23, stderr="simulated package install failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
text = _command_text(argv)
|
||||||
|
if "govoplan_core.commands.init_db" in text:
|
||||||
|
return _completed(24, stderr="simulated migration failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace:
|
||||||
|
if kwargs.get("shell") and str(argv).strip() == "restart-fails":
|
||||||
|
return _completed(25, stderr="simulated restart failure")
|
||||||
|
return _success_run(argv, *_args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
ORIGINAL_SUBPROCESS_RUN = subprocess.run
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]:
|
||||||
|
import govoplan_core.core.module_installer as installer
|
||||||
|
|
||||||
|
original = installer.subprocess.run
|
||||||
|
installer.subprocess.run = fake_run # type: ignore[assignment]
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
installer.subprocess.run = original # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _patch_urlopen() -> Iterable[None]:
|
||||||
|
import govoplan_core.core.module_installer as installer
|
||||||
|
|
||||||
|
original = installer.urllib.request.urlopen
|
||||||
|
|
||||||
|
def fail(*_args: object, **_kwargs: object) -> None:
|
||||||
|
raise URLError("simulated health timeout")
|
||||||
|
|
||||||
|
installer.urllib.request.urlopen = fail # type: ignore[assignment]
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
installer.urllib.request.urlopen = original # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
SCENARIOS: dict[str, Scenario] = {
|
||||||
|
"destructive-retirement-failure": destructive_retirement_failure,
|
||||||
|
"health-timeout": health_timeout,
|
||||||
|
"migration-failure": migration_failure,
|
||||||
|
"package-failure": package_failure,
|
||||||
|
"postgres-hooks": postgres_hooks,
|
||||||
|
"queue-and-lock": queue_and_lock,
|
||||||
|
"restart-failure": restart_failure,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
128
scripts/postgres-integration-check.py
Normal file
128
scripts/postgres-integration-check.py
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SRC = ROOT / "src"
|
||||||
|
|
||||||
|
DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = (
|
||||||
|
("core-only", "tenancy,access,admin"),
|
||||||
|
("files-only", "tenancy,access,admin,files"),
|
||||||
|
("mail-only", "tenancy,access,admin,mail"),
|
||||||
|
("campaign-only", "tenancy,access,admin,campaigns"),
|
||||||
|
("campaign-with-files", "tenancy,access,admin,campaigns,files"),
|
||||||
|
("campaign-with-mail", "tenancy,access,admin,campaigns,mail"),
|
||||||
|
("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--database-url",
|
||||||
|
default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"),
|
||||||
|
help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--module-set",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
metavar="NAME=MODULES",
|
||||||
|
help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--reset-schema",
|
||||||
|
action="store_true",
|
||||||
|
help="DROP and recreate the public schema before every module set. Use only with a disposable database.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.")
|
||||||
|
parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.")
|
||||||
|
parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.")
|
||||||
|
parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
database_url = (args.database_url or "").strip()
|
||||||
|
if not database_url:
|
||||||
|
parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required")
|
||||||
|
if not database_url.startswith("postgresql"):
|
||||||
|
parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.")
|
||||||
|
|
||||||
|
module_sets = _parse_module_sets(args.module_set)
|
||||||
|
ok = True
|
||||||
|
for name, modules in module_sets:
|
||||||
|
print(f"\n== {name}: {modules} ==")
|
||||||
|
if args.reset_schema:
|
||||||
|
_reset_public_schema(database_url)
|
||||||
|
env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env)
|
||||||
|
if not args.skip_migrate:
|
||||||
|
ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok
|
||||||
|
if not args.skip_smoke:
|
||||||
|
ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]:
|
||||||
|
if not raw:
|
||||||
|
return DEFAULT_MODULE_SETS
|
||||||
|
parsed: list[tuple[str, str]] = []
|
||||||
|
for item in raw:
|
||||||
|
name, separator, modules = item.partition("=")
|
||||||
|
if not separator or not name.strip() or not modules.strip():
|
||||||
|
raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.")
|
||||||
|
parsed.append((name.strip(), modules.strip()))
|
||||||
|
return tuple(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]:
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["APP_ENV"] = app_env
|
||||||
|
env["DATABASE_URL"] = database_url
|
||||||
|
env["ENABLED_MODULES"] = modules
|
||||||
|
env["DEV_AUTO_MIGRATE_ENABLED"] = "false"
|
||||||
|
env["DEV_BOOTSTRAP_ENABLED"] = "false"
|
||||||
|
env["CELERY_ENABLED"] = "false"
|
||||||
|
env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii"))
|
||||||
|
env.setdefault("FILE_STORAGE_BACKEND", "local")
|
||||||
|
env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files"))
|
||||||
|
env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox"))
|
||||||
|
env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part)
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def _run(command: list[str], *, env: dict[str, str]) -> bool:
|
||||||
|
print("+ " + " ".join(command))
|
||||||
|
result = subprocess.run(command, cwd=ROOT, env=env, check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_public_schema(database_url: str) -> None:
|
||||||
|
try:
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
except ModuleNotFoundError as exc:
|
||||||
|
raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc
|
||||||
|
|
||||||
|
engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
|
||||||
|
try:
|
||||||
|
with engine.connect() as connection:
|
||||||
|
connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE")
|
||||||
|
connection.exec_driver_sql("CREATE SCHEMA public")
|
||||||
|
connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public")
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
print("Reset PostgreSQL schema: public")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -24,6 +24,11 @@ Options:
|
|||||||
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
-m, --message <text> Commit message. Defaults to "Release v<x.y.z>".
|
||||||
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
--tag-message <text> Annotated tag message. Defaults to the commit message.
|
||||||
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
--skip-release-lock Do not regenerate core webui/package-lock.release.json.
|
||||||
|
--warn-migration-audit Run the migration release audit without baseline strictness.
|
||||||
|
--strict-migration-audit
|
||||||
|
Require current Alembic heads to be recorded in the
|
||||||
|
latest migration release baseline.
|
||||||
|
--skip-migration-audit Do not run the release migration audit.
|
||||||
--publish-web-catalog After core/modules are pushed, publish a signed release
|
--publish-web-catalog After core/modules are pushed, publish a signed release
|
||||||
catalog into govoplan-web.
|
catalog into govoplan-web.
|
||||||
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
--web-root <path> govoplan-web checkout for --publish-web-catalog.
|
||||||
@@ -51,18 +56,22 @@ Repos:
|
|||||||
govoplan-access
|
govoplan-access
|
||||||
govoplan-admin
|
govoplan-admin
|
||||||
govoplan-tenancy
|
govoplan-tenancy
|
||||||
|
govoplan-organizations
|
||||||
|
govoplan-identity
|
||||||
govoplan-policy
|
govoplan-policy
|
||||||
govoplan-audit
|
govoplan-audit
|
||||||
|
govoplan-dashboard
|
||||||
govoplan-files
|
govoplan-files
|
||||||
govoplan-mail
|
govoplan-mail
|
||||||
govoplan-campaign
|
govoplan-campaign
|
||||||
govoplan-calendar
|
govoplan-calendar
|
||||||
|
govoplan-ops
|
||||||
govoplan-core
|
govoplan-core
|
||||||
|
|
||||||
Roadmap/scaffold module repositories are tag-only until they have package
|
Roadmap/scaffold module repositories are tag-only until they have package
|
||||||
metadata: addresses, appointments, cases, connectors, dms, erp,
|
metadata: addresses, appointments, cases, connectors, dms, erp,
|
||||||
fit-connect, forms, identity-trust, idm, ledger, notifications, ops,
|
fit-connect, forms, identity-trust, idm, ledger, notifications, payments,
|
||||||
payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
portal, reporting, scheduling, search, tasks, templates, workflow, xoev,
|
||||||
xrechnung, and xta-osci.
|
xrechnung, and xta-osci.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
@@ -83,12 +92,16 @@ PACKAGE_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-access"
|
"$PARENT/govoplan-access"
|
||||||
"$PARENT/govoplan-admin"
|
"$PARENT/govoplan-admin"
|
||||||
"$PARENT/govoplan-tenancy"
|
"$PARENT/govoplan-tenancy"
|
||||||
|
"$PARENT/govoplan-organizations"
|
||||||
|
"$PARENT/govoplan-identity"
|
||||||
"$PARENT/govoplan-policy"
|
"$PARENT/govoplan-policy"
|
||||||
"$PARENT/govoplan-audit"
|
"$PARENT/govoplan-audit"
|
||||||
|
"$PARENT/govoplan-dashboard"
|
||||||
"$PARENT/govoplan-files"
|
"$PARENT/govoplan-files"
|
||||||
"$PARENT/govoplan-mail"
|
"$PARENT/govoplan-mail"
|
||||||
"$PARENT/govoplan-campaign"
|
"$PARENT/govoplan-campaign"
|
||||||
"$PARENT/govoplan-calendar"
|
"$PARENT/govoplan-calendar"
|
||||||
|
"$PARENT/govoplan-ops"
|
||||||
)
|
)
|
||||||
TAG_ONLY_MODULE_REPOS=(
|
TAG_ONLY_MODULE_REPOS=(
|
||||||
"$PARENT/govoplan-addresses"
|
"$PARENT/govoplan-addresses"
|
||||||
@@ -103,7 +116,6 @@ TAG_ONLY_MODULE_REPOS=(
|
|||||||
"$PARENT/govoplan-idm"
|
"$PARENT/govoplan-idm"
|
||||||
"$PARENT/govoplan-ledger"
|
"$PARENT/govoplan-ledger"
|
||||||
"$PARENT/govoplan-notifications"
|
"$PARENT/govoplan-notifications"
|
||||||
"$PARENT/govoplan-ops"
|
|
||||||
"$PARENT/govoplan-payments"
|
"$PARENT/govoplan-payments"
|
||||||
"$PARENT/govoplan-portal"
|
"$PARENT/govoplan-portal"
|
||||||
"$PARENT/govoplan-reporting"
|
"$PARENT/govoplan-reporting"
|
||||||
@@ -132,6 +144,7 @@ TAG_MESSAGE=""
|
|||||||
DRY_RUN=0
|
DRY_RUN=0
|
||||||
YES=0
|
YES=0
|
||||||
GENERATE_RELEASE_LOCK=1
|
GENERATE_RELEASE_LOCK=1
|
||||||
|
MIGRATION_AUDIT_MODE="auto"
|
||||||
NPM_BIN="${NPM:-}"
|
NPM_BIN="${NPM:-}"
|
||||||
PUBLISH_WEB_CATALOG=0
|
PUBLISH_WEB_CATALOG=0
|
||||||
WEB_ROOT="$PARENT/govoplan-web"
|
WEB_ROOT="$PARENT/govoplan-web"
|
||||||
@@ -282,6 +295,12 @@ update_manifest_version() {
|
|||||||
govoplan-tenancy)
|
govoplan-tenancy)
|
||||||
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py"
|
||||||
;;
|
;;
|
||||||
|
govoplan-organizations)
|
||||||
|
manifest_path="$repo/src/govoplan_organizations/backend/manifest.py"
|
||||||
|
;;
|
||||||
|
govoplan-identity)
|
||||||
|
manifest_path="$repo/src/govoplan_identity/backend/manifest.py"
|
||||||
|
;;
|
||||||
govoplan-policy)
|
govoplan-policy)
|
||||||
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
manifest_path="$repo/src/govoplan_policy/backend/manifest.py"
|
||||||
;;
|
;;
|
||||||
@@ -408,6 +427,32 @@ generate_release_lock() {
|
|||||||
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_migration_release_audit() {
|
||||||
|
local audit_script="$ROOT/scripts/release-migration-audit.py"
|
||||||
|
local command=("$PYTHON" "$audit_script")
|
||||||
|
|
||||||
|
case "$MIGRATION_AUDIT_MODE" in
|
||||||
|
skip)
|
||||||
|
echo "Migration release audit: skipped"
|
||||||
|
return
|
||||||
|
;;
|
||||||
|
strict)
|
||||||
|
command+=("--strict")
|
||||||
|
;;
|
||||||
|
auto)
|
||||||
|
command+=("--strict-if-baseline")
|
||||||
|
;;
|
||||||
|
warn)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
[[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script"
|
||||||
|
run "${command[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
print_command() {
|
print_command() {
|
||||||
printf '+'
|
printf '+'
|
||||||
printf ' %q' "$@"
|
printf ' %q' "$@"
|
||||||
@@ -514,6 +559,18 @@ while [[ $# -gt 0 ]]; do
|
|||||||
GENERATE_RELEASE_LOCK=0
|
GENERATE_RELEASE_LOCK=0
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
|
--warn-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="warn"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--strict-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="strict"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--skip-migration-audit)
|
||||||
|
MIGRATION_AUDIT_MODE="skip"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
--publish-web-catalog)
|
--publish-web-catalog)
|
||||||
PUBLISH_WEB_CATALOG=1
|
PUBLISH_WEB_CATALOG=1
|
||||||
shift
|
shift
|
||||||
@@ -690,6 +747,7 @@ if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then
|
|||||||
else
|
else
|
||||||
echo " release lock: skipped"
|
echo " release lock: skipped"
|
||||||
fi
|
fi
|
||||||
|
echo " migration audit: $MIGRATION_AUDIT_MODE"
|
||||||
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then
|
||||||
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed"
|
||||||
else
|
else
|
||||||
@@ -707,6 +765,8 @@ for repo in "${REPOS[@]}"; do
|
|||||||
echo
|
echo
|
||||||
done
|
done
|
||||||
|
|
||||||
|
run_migration_release_audit
|
||||||
|
|
||||||
confirm_release
|
confirm_release
|
||||||
|
|
||||||
for repo in "${PACKAGE_REPOS[@]}"; do
|
for repo in "${PACKAGE_REPOS[@]}"; do
|
||||||
|
|||||||
422
scripts/release-migration-audit.py
Normal file
422
scripts/release-migration-audit.py
Normal file
@@ -0,0 +1,422 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import ast
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Migration:
|
||||||
|
owner: str
|
||||||
|
path: Path
|
||||||
|
revision: str
|
||||||
|
down_revisions: tuple[str, ...]
|
||||||
|
branch_labels: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Audit GovOPlaN Alembic migrations against the release baseline policy.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--workspace-root",
|
||||||
|
type=Path,
|
||||||
|
default=ROOT.parent,
|
||||||
|
help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--baseline-file",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_BASELINE_FILE,
|
||||||
|
help="JSON file recording released migration heads.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict",
|
||||||
|
action="store_true",
|
||||||
|
help="Fail when current heads are not recorded in the latest release baseline.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict-if-baseline",
|
||||||
|
action="store_true",
|
||||||
|
help="Run in strict mode only after at least one release baseline is recorded.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--record-release",
|
||||||
|
metavar="VERSION",
|
||||||
|
help="Record the current reviewed migration heads as a release baseline.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--replace-release",
|
||||||
|
action="store_true",
|
||||||
|
help="Replace an existing release entry when used with --record-release.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--squash-plan",
|
||||||
|
action="store_true",
|
||||||
|
help="Print the reviewed/manual squash checklist for the current graph.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
migrations = discover_migrations(args.workspace_root)
|
||||||
|
baseline = load_baseline_file(args.baseline_file)
|
||||||
|
report = build_report(
|
||||||
|
migrations,
|
||||||
|
baseline=baseline,
|
||||||
|
baseline_file=args.baseline_file,
|
||||||
|
workspace_root=args.workspace_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.record_release:
|
||||||
|
if report["graph_errors"]:
|
||||||
|
print_text_report(report)
|
||||||
|
return 1
|
||||||
|
baseline = record_release_baseline(
|
||||||
|
baseline,
|
||||||
|
report,
|
||||||
|
release=args.record_release,
|
||||||
|
replace=args.replace_release,
|
||||||
|
)
|
||||||
|
write_baseline_file(args.baseline_file, baseline)
|
||||||
|
report = build_report(
|
||||||
|
migrations,
|
||||||
|
baseline=baseline,
|
||||||
|
baseline_file=args.baseline_file,
|
||||||
|
workspace_root=args.workspace_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.squash_plan:
|
||||||
|
print_squash_plan(report)
|
||||||
|
elif args.json:
|
||||||
|
print(json.dumps(report, indent=2, sort_keys=True))
|
||||||
|
else:
|
||||||
|
print_text_report(report)
|
||||||
|
|
||||||
|
has_graph_errors = bool(report["graph_errors"])
|
||||||
|
has_strict_errors = bool(report["strict_errors"])
|
||||||
|
strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0)
|
||||||
|
if has_graph_errors or (strict_required and has_strict_errors):
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def discover_migrations(workspace_root: Path) -> list[Migration]:
|
||||||
|
migrations: list[Migration] = []
|
||||||
|
for versions_dir in _migration_version_dirs(workspace_root):
|
||||||
|
owner = owner_for_versions_dir(versions_dir)
|
||||||
|
for path in sorted(versions_dir.glob("*.py")):
|
||||||
|
if path.name == "__init__.py":
|
||||||
|
continue
|
||||||
|
migration = parse_migration_file(owner, path)
|
||||||
|
if migration is not None:
|
||||||
|
migrations.append(migration)
|
||||||
|
return migrations
|
||||||
|
|
||||||
|
|
||||||
|
def _migration_version_dirs(workspace_root: Path) -> list[Path]:
|
||||||
|
candidates = [ROOT / "alembic" / "versions"]
|
||||||
|
for repo in sorted(workspace_root.glob("govoplan-*")):
|
||||||
|
candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions")))
|
||||||
|
seen: set[Path] = set()
|
||||||
|
result: list[Path] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
resolved = candidate.resolve()
|
||||||
|
if resolved in seen or not candidate.is_dir():
|
||||||
|
continue
|
||||||
|
seen.add(resolved)
|
||||||
|
result.append(candidate)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def owner_for_versions_dir(versions_dir: Path) -> str:
|
||||||
|
if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve():
|
||||||
|
return "govoplan-core"
|
||||||
|
for parent in versions_dir.parents:
|
||||||
|
if parent.name.startswith("govoplan-"):
|
||||||
|
return parent.name
|
||||||
|
return versions_dir.parent.name
|
||||||
|
|
||||||
|
|
||||||
|
def parse_migration_file(owner: str, path: Path) -> Migration | None:
|
||||||
|
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||||
|
values: dict[str, Any] = {}
|
||||||
|
for statement in tree.body:
|
||||||
|
if isinstance(statement, ast.Assign):
|
||||||
|
for target in statement.targets:
|
||||||
|
if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}:
|
||||||
|
values[target.id] = ast.literal_eval(statement.value)
|
||||||
|
elif (
|
||||||
|
isinstance(statement, ast.AnnAssign)
|
||||||
|
and isinstance(statement.target, ast.Name)
|
||||||
|
and statement.target.id in {"revision", "down_revision", "branch_labels"}
|
||||||
|
and statement.value is not None
|
||||||
|
):
|
||||||
|
values[statement.target.id] = ast.literal_eval(statement.value)
|
||||||
|
revision = values.get("revision")
|
||||||
|
if not isinstance(revision, str):
|
||||||
|
return None
|
||||||
|
return Migration(
|
||||||
|
owner=owner,
|
||||||
|
path=path,
|
||||||
|
revision=revision,
|
||||||
|
down_revisions=_normalize_revision_tuple(values.get("down_revision")),
|
||||||
|
branch_labels=_normalize_string_tuple(values.get("branch_labels")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_revision_tuple(value: Any) -> tuple[str, ...]:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(value, str):
|
||||||
|
return (value,)
|
||||||
|
if isinstance(value, (list, tuple, set)):
|
||||||
|
return tuple(str(item) for item in value if item is not None)
|
||||||
|
return (str(value),)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_string_tuple(value: Any) -> tuple[str, ...]:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(value, str):
|
||||||
|
return (value,)
|
||||||
|
if isinstance(value, (list, tuple, set)):
|
||||||
|
return tuple(str(item) for item in value)
|
||||||
|
return (str(value),)
|
||||||
|
|
||||||
|
|
||||||
|
def load_baseline_file(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.exists():
|
||||||
|
return {"version": 1, "releases": [], "_missing": True}
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise SystemExit(f"{path} must contain a JSON object")
|
||||||
|
releases = payload.get("releases")
|
||||||
|
if not isinstance(releases, list):
|
||||||
|
raise SystemExit(f"{path} must contain a releases list")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def build_report(
|
||||||
|
migrations: list[Migration],
|
||||||
|
*,
|
||||||
|
baseline: dict[str, Any],
|
||||||
|
baseline_file: Path,
|
||||||
|
workspace_root: Path,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
revisions: dict[str, Migration] = {}
|
||||||
|
graph_errors: list[str] = []
|
||||||
|
for migration in migrations:
|
||||||
|
existing = revisions.get(migration.revision)
|
||||||
|
if existing is not None:
|
||||||
|
graph_errors.append(
|
||||||
|
f"duplicate revision {migration.revision}: {existing.path} and {migration.path}"
|
||||||
|
)
|
||||||
|
revisions[migration.revision] = migration
|
||||||
|
|
||||||
|
referenced = {revision for migration in migrations for revision in migration.down_revisions}
|
||||||
|
for migration in migrations:
|
||||||
|
for down_revision in migration.down_revisions:
|
||||||
|
if down_revision not in revisions:
|
||||||
|
graph_errors.append(
|
||||||
|
f"{migration.revision} references missing down_revision {down_revision} in {migration.path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
current_heads = sorted(revision for revision in revisions if revision not in referenced)
|
||||||
|
current_head_entries = [
|
||||||
|
{
|
||||||
|
"owner": revisions[revision].owner,
|
||||||
|
"revision": revision,
|
||||||
|
}
|
||||||
|
for revision in current_heads
|
||||||
|
]
|
||||||
|
owner_rows = []
|
||||||
|
for owner in sorted({migration.owner for migration in migrations}):
|
||||||
|
owner_migrations = [migration for migration in migrations if migration.owner == owner]
|
||||||
|
owner_revisions = {migration.revision for migration in owner_migrations}
|
||||||
|
owner_referenced = {
|
||||||
|
revision
|
||||||
|
for migration in owner_migrations
|
||||||
|
for revision in migration.down_revisions
|
||||||
|
if revision in owner_revisions
|
||||||
|
}
|
||||||
|
owner_heads = sorted(owner_revisions - owner_referenced)
|
||||||
|
owner_rows.append(
|
||||||
|
{
|
||||||
|
"owner": owner,
|
||||||
|
"migrations": len(owner_migrations),
|
||||||
|
"heads": owner_heads,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
releases = baseline.get("releases") or []
|
||||||
|
latest_release = releases[-1] if releases else None
|
||||||
|
latest_heads = _latest_release_heads(latest_release)
|
||||||
|
unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads
|
||||||
|
|
||||||
|
strict_errors: list[str] = []
|
||||||
|
if baseline.get("_missing"):
|
||||||
|
strict_errors.append(f"baseline file is missing: {baseline_file}")
|
||||||
|
if not releases:
|
||||||
|
strict_errors.append("no released migration baseline has been recorded yet")
|
||||||
|
if unrecorded_heads:
|
||||||
|
strict_errors.append(
|
||||||
|
"current migration heads are not recorded in the latest release baseline: "
|
||||||
|
+ ", ".join(unrecorded_heads)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"workspace_root": str(workspace_root),
|
||||||
|
"baseline_file": str(baseline_file),
|
||||||
|
"baseline_file_missing": bool(baseline.get("_missing")),
|
||||||
|
"release_count": len(releases),
|
||||||
|
"latest_release": latest_release,
|
||||||
|
"owners": owner_rows,
|
||||||
|
"current_heads": current_heads,
|
||||||
|
"current_head_entries": current_head_entries,
|
||||||
|
"graph_errors": graph_errors,
|
||||||
|
"strict_errors": strict_errors,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_release_heads(latest_release: Any) -> set[str]:
|
||||||
|
if not isinstance(latest_release, dict):
|
||||||
|
return set()
|
||||||
|
heads = latest_release.get("heads") or []
|
||||||
|
graph_heads = latest_release.get("graph_heads") or []
|
||||||
|
result: set[str] = set()
|
||||||
|
for head_list in (heads, graph_heads):
|
||||||
|
if not isinstance(head_list, list):
|
||||||
|
continue
|
||||||
|
for entry in head_list:
|
||||||
|
if isinstance(entry, str):
|
||||||
|
result.add(entry)
|
||||||
|
elif isinstance(entry, dict) and isinstance(entry.get("revision"), str):
|
||||||
|
result.add(entry["revision"])
|
||||||
|
owner_heads = latest_release.get("owner_heads") or []
|
||||||
|
if isinstance(owner_heads, list):
|
||||||
|
for entry in owner_heads:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
revisions = entry.get("revisions") or []
|
||||||
|
if isinstance(revisions, str):
|
||||||
|
result.add(revisions)
|
||||||
|
elif isinstance(revisions, list):
|
||||||
|
result.update(revision for revision in revisions if isinstance(revision, str))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def record_release_baseline(
|
||||||
|
baseline: dict[str, Any],
|
||||||
|
report: dict[str, Any],
|
||||||
|
*,
|
||||||
|
release: str,
|
||||||
|
replace: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
payload = {
|
||||||
|
key: value
|
||||||
|
for key, value in baseline.items()
|
||||||
|
if not key.startswith("_")
|
||||||
|
}
|
||||||
|
payload.setdefault("version", 1)
|
||||||
|
releases = list(payload.get("releases") or [])
|
||||||
|
existing_index = next(
|
||||||
|
(index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if existing_index is not None and not replace:
|
||||||
|
raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it")
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"release": release,
|
||||||
|
"recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||||
|
"squash_policy": "reviewed-manual",
|
||||||
|
"heads": report["current_head_entries"],
|
||||||
|
"owner_heads": [
|
||||||
|
{
|
||||||
|
"owner": owner["owner"],
|
||||||
|
"revisions": owner["heads"],
|
||||||
|
}
|
||||||
|
for owner in report["owners"]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
if existing_index is None:
|
||||||
|
releases.append(entry)
|
||||||
|
else:
|
||||||
|
releases[existing_index] = entry
|
||||||
|
payload["releases"] = releases
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def print_text_report(report: dict[str, Any]) -> None:
|
||||||
|
print("Migration release audit")
|
||||||
|
print(f" workspace: {report['workspace_root']}")
|
||||||
|
print(f" baseline file: {report['baseline_file']}")
|
||||||
|
print(f" recorded releases: {report['release_count']}")
|
||||||
|
print()
|
||||||
|
print("Owners:")
|
||||||
|
for owner in report["owners"]:
|
||||||
|
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||||
|
print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}")
|
||||||
|
print()
|
||||||
|
print("Current heads:")
|
||||||
|
for entry in report["current_head_entries"]:
|
||||||
|
print(f" {entry['owner']}: {entry['revision']}")
|
||||||
|
if not report["current_head_entries"]:
|
||||||
|
print(" -")
|
||||||
|
|
||||||
|
if report["graph_errors"]:
|
||||||
|
print()
|
||||||
|
print("Graph errors:")
|
||||||
|
for error in report["graph_errors"]:
|
||||||
|
print(f" - {error}")
|
||||||
|
|
||||||
|
if report["strict_errors"]:
|
||||||
|
print()
|
||||||
|
print("Release baseline warnings:")
|
||||||
|
for error in report["strict_errors"]:
|
||||||
|
print(f" - {error}")
|
||||||
|
|
||||||
|
|
||||||
|
def print_squash_plan(report: dict[str, Any]) -> None:
|
||||||
|
print("Manual migration squash plan")
|
||||||
|
print()
|
||||||
|
print("Policy:")
|
||||||
|
print(" - Do not rewrite revision IDs that have shipped to real installations.")
|
||||||
|
print(" - Squash only unreleased development migrations.")
|
||||||
|
print(" - Review generated baseline/upgrade migrations like normal schema code.")
|
||||||
|
print(" - Run PostgreSQL migration smoke checks after any squash.")
|
||||||
|
print()
|
||||||
|
print("Current owner heads:")
|
||||||
|
for owner in report["owners"]:
|
||||||
|
heads = ", ".join(owner["heads"]) if owner["heads"] else "-"
|
||||||
|
print(f" - {owner['owner']}: {heads}")
|
||||||
|
print()
|
||||||
|
print("Current Alembic graph heads:")
|
||||||
|
for entry in report["current_head_entries"]:
|
||||||
|
print(f" - {entry['owner']}: {entry['revision']}")
|
||||||
|
if not report["current_head_entries"]:
|
||||||
|
print(" -")
|
||||||
|
print()
|
||||||
|
print("Release steps:")
|
||||||
|
print(" 1. Decide which migration revisions are unreleased and may be folded.")
|
||||||
|
print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.")
|
||||||
|
print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.")
|
||||||
|
print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release <x.y.z>.")
|
||||||
|
print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -9,7 +9,7 @@ from govoplan_core.db.base import Base, TimestampMixin
|
|||||||
|
|
||||||
|
|
||||||
class SystemSettings(Base, TimestampMixin):
|
class SystemSettings(Base, TimestampMixin):
|
||||||
__tablename__ = "system_settings"
|
__tablename__ = "core_system_settings"
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global")
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global")
|
||||||
default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False)
|
default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False)
|
||||||
|
|||||||
@@ -24,6 +24,21 @@ class AuditLogListResponse(BaseModel):
|
|||||||
items: list[AuditLogItemResponse]
|
items: list[AuditLogItemResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class DeltaDeletedItem(BaseModel):
|
||||||
|
id: str
|
||||||
|
resource_type: str | None = None
|
||||||
|
revision: str | None = None
|
||||||
|
deleted_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DeltaCollectionResponse(BaseModel):
|
||||||
|
items: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
|
deleted: list[DeltaDeletedItem] = Field(default_factory=list)
|
||||||
|
watermark: str | None = None
|
||||||
|
has_more: bool = False
|
||||||
|
full: bool = False
|
||||||
|
|
||||||
|
|
||||||
class LoginRequest(BaseModel):
|
class LoginRequest(BaseModel):
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
@@ -46,6 +61,7 @@ class TenantInfo(BaseModel):
|
|||||||
name: str
|
name: str
|
||||||
is_active: bool = True
|
is_active: bool = True
|
||||||
default_locale: str = "en"
|
default_locale: str = "en"
|
||||||
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class TenantMembershipInfo(TenantInfo):
|
class TenantMembershipInfo(TenantInfo):
|
||||||
@@ -53,6 +69,16 @@ class TenantMembershipInfo(TenantInfo):
|
|||||||
is_active: bool = True
|
is_active: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class UserUiPreferences(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="ignore")
|
||||||
|
|
||||||
|
compact_tables: bool = False
|
||||||
|
show_inline_help_hints: bool = True
|
||||||
|
reduce_motion: bool = False
|
||||||
|
sticky_section_sidebars: bool = True
|
||||||
|
theme: Literal["system", "light", "dark"] = "system"
|
||||||
|
|
||||||
|
|
||||||
class UserInfo(BaseModel):
|
class UserInfo(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
account_id: str
|
account_id: str
|
||||||
@@ -63,6 +89,9 @@ class UserInfo(BaseModel):
|
|||||||
tenant_display_name: str | None = None
|
tenant_display_name: str | None = None
|
||||||
is_tenant_admin: bool = False
|
is_tenant_admin: bool = False
|
||||||
password_reset_required: bool = False
|
password_reset_required: bool = False
|
||||||
|
preferred_language: str | None = None
|
||||||
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
|
ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences)
|
||||||
|
|
||||||
|
|
||||||
class ProfileUpdateRequest(BaseModel):
|
class ProfileUpdateRequest(BaseModel):
|
||||||
@@ -70,6 +99,15 @@ class ProfileUpdateRequest(BaseModel):
|
|||||||
|
|
||||||
display_name: str | None = Field(default=None, max_length=255)
|
display_name: str | None = Field(default=None, max_length=255)
|
||||||
tenant_display_name: str | None = Field(default=None, max_length=255)
|
tenant_display_name: str | None = Field(default=None, max_length=255)
|
||||||
|
preferred_language: str | None = Field(default=None, max_length=20)
|
||||||
|
enabled_language_codes: list[str] | None = None
|
||||||
|
ui_preferences: UserUiPreferences | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class LanguageInfo(BaseModel):
|
||||||
|
code: str
|
||||||
|
label: str
|
||||||
|
native_label: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class RoleInfo(BaseModel):
|
class RoleInfo(BaseModel):
|
||||||
@@ -86,6 +124,25 @@ class GroupInfo(BaseModel):
|
|||||||
name: str
|
name: str
|
||||||
|
|
||||||
|
|
||||||
|
class PrincipalContextInfo(BaseModel):
|
||||||
|
account_id: str
|
||||||
|
membership_id: str | None = None
|
||||||
|
tenant_id: str | None = None
|
||||||
|
identity_id: str | None = None
|
||||||
|
scopes: list[str] = Field(default_factory=list)
|
||||||
|
group_ids: list[str] = Field(default_factory=list)
|
||||||
|
role_ids: list[str] = Field(default_factory=list)
|
||||||
|
function_assignment_ids: list[str] = Field(default_factory=list)
|
||||||
|
delegation_ids: list[str] = Field(default_factory=list)
|
||||||
|
auth_method: Literal["session", "api_key", "service_account"] = "session"
|
||||||
|
api_key_id: str | None = None
|
||||||
|
session_id: str | None = None
|
||||||
|
service_account_id: str | None = None
|
||||||
|
acting_for_account_id: str | None = None
|
||||||
|
email: str | None = None
|
||||||
|
display_name: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class LoginResponse(BaseModel):
|
class LoginResponse(BaseModel):
|
||||||
access_token: str
|
access_token: str
|
||||||
token_type: str = "bearer"
|
token_type: str = "bearer"
|
||||||
@@ -98,6 +155,10 @@ class LoginResponse(BaseModel):
|
|||||||
scopes: list[str]
|
scopes: list[str]
|
||||||
roles: list[RoleInfo] = Field(default_factory=list)
|
roles: list[RoleInfo] = Field(default_factory=list)
|
||||||
groups: list[GroupInfo] = Field(default_factory=list)
|
groups: list[GroupInfo] = Field(default_factory=list)
|
||||||
|
principal: PrincipalContextInfo | None = None
|
||||||
|
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||||
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
|
default_language: str = "en"
|
||||||
|
|
||||||
|
|
||||||
class MeResponse(BaseModel):
|
class MeResponse(BaseModel):
|
||||||
@@ -109,3 +170,7 @@ class MeResponse(BaseModel):
|
|||||||
scopes: list[str]
|
scopes: list[str]
|
||||||
roles: list[RoleInfo] = Field(default_factory=list)
|
roles: list[RoleInfo] = Field(default_factory=list)
|
||||||
groups: list[GroupInfo] = Field(default_factory=list)
|
groups: list[GroupInfo] = Field(default_factory=list)
|
||||||
|
principal: PrincipalContextInfo | None = None
|
||||||
|
available_languages: list[LanguageInfo] = Field(default_factory=list)
|
||||||
|
enabled_language_codes: list[str] = Field(default_factory=list)
|
||||||
|
default_language: str = "en"
|
||||||
|
|||||||
@@ -1,14 +1,21 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_access.backend.auth.dependencies import ApiPrincipal
|
from govoplan_access.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.change_sequence import record_change
|
||||||
|
from govoplan_core.core.events import current_event_trace, normalize_trace_id
|
||||||
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
from govoplan_core.privacy.retention import sanitize_audit_details_for_policy
|
||||||
from govoplan_audit.backend.db.models import AuditLog
|
from govoplan_audit.backend.db.models import AuditLog
|
||||||
|
|
||||||
|
|
||||||
|
AUDIT_MODULE_ID = "audit"
|
||||||
|
AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events"
|
||||||
|
AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events"
|
||||||
|
|
||||||
SENSITIVE_DETAIL_KEYS = {
|
SENSITIVE_DETAIL_KEYS = {
|
||||||
"password",
|
"password",
|
||||||
"smtp_password",
|
"smtp_password",
|
||||||
@@ -16,9 +23,40 @@ SENSITIVE_DETAIL_KEYS = {
|
|||||||
"secret",
|
"secret",
|
||||||
"api_key",
|
"api_key",
|
||||||
"token",
|
"token",
|
||||||
|
"access_token",
|
||||||
|
"refresh_token",
|
||||||
|
"authorization",
|
||||||
|
"cookie",
|
||||||
|
"credentials",
|
||||||
|
"credential",
|
||||||
|
"private_key",
|
||||||
|
"claim_token",
|
||||||
|
"build_token",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_text(value: object | None) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
candidate = str(value).strip()
|
||||||
|
return candidate or None
|
||||||
|
|
||||||
|
|
||||||
|
def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]:
|
||||||
|
if not isinstance(trace, Mapping):
|
||||||
|
return {}
|
||||||
|
compact: dict[str, str] = {}
|
||||||
|
correlation_id = _optional_text(trace.get("correlation_id"))
|
||||||
|
causation_id = _optional_text(trace.get("causation_id"))
|
||||||
|
clean_correlation_id = normalize_trace_id(correlation_id)
|
||||||
|
clean_causation_id = normalize_trace_id(causation_id)
|
||||||
|
if clean_correlation_id:
|
||||||
|
compact["correlation_id"] = clean_correlation_id
|
||||||
|
if clean_causation_id:
|
||||||
|
compact["causation_id"] = clean_causation_id
|
||||||
|
return compact
|
||||||
|
|
||||||
|
|
||||||
def _sanitize_details(value: Any) -> Any:
|
def _sanitize_details(value: Any) -> Any:
|
||||||
if isinstance(value, dict):
|
if isinstance(value, dict):
|
||||||
sanitized: dict[str, Any] = {}
|
sanitized: dict[str, Any] = {}
|
||||||
@@ -33,6 +71,70 @@ def _sanitize_details(value: Any) -> Any:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def audit_operation_context(
|
||||||
|
*,
|
||||||
|
module_id: object | None = None,
|
||||||
|
request_id: object | None = None,
|
||||||
|
run_id: object | None = None,
|
||||||
|
outcome: object | None = None,
|
||||||
|
trace: Mapping[str, object] | None = None,
|
||||||
|
**details: Any,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Build concise operational audit details for admin and lifecycle actions."""
|
||||||
|
|
||||||
|
payload: dict[str, Any] = {}
|
||||||
|
for key, value in (
|
||||||
|
("module_id", module_id),
|
||||||
|
("request_id", request_id),
|
||||||
|
("run_id", run_id),
|
||||||
|
("outcome", outcome),
|
||||||
|
):
|
||||||
|
text = _optional_text(value)
|
||||||
|
if text is not None:
|
||||||
|
payload[key] = text
|
||||||
|
for key, value in details.items():
|
||||||
|
if value is not None:
|
||||||
|
payload[key] = _sanitize_details(value)
|
||||||
|
compact_trace = _compact_trace(trace)
|
||||||
|
if compact_trace:
|
||||||
|
existing = payload.get("_trace")
|
||||||
|
payload["_trace"] = {**existing, **compact_trace} if isinstance(existing, dict) else compact_trace
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _trace_details(
|
||||||
|
details: dict[str, Any],
|
||||||
|
*,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
causation_id: str | None = None,
|
||||||
|
) -> tuple[dict[str, Any], dict[str, str]]:
|
||||||
|
active_trace = current_event_trace()
|
||||||
|
trace: dict[str, str] = {}
|
||||||
|
clean_correlation_id = normalize_trace_id(correlation_id)
|
||||||
|
clean_causation_id = normalize_trace_id(causation_id)
|
||||||
|
if clean_correlation_id or active_trace:
|
||||||
|
trace["correlation_id"] = clean_correlation_id or active_trace.correlation_id
|
||||||
|
if clean_causation_id or (active_trace and active_trace.causation_id):
|
||||||
|
trace["causation_id"] = clean_causation_id or str(active_trace.causation_id)
|
||||||
|
if not trace:
|
||||||
|
return details, {}
|
||||||
|
merged = dict(details)
|
||||||
|
existing = merged.get("_trace")
|
||||||
|
merged["_trace"] = {**existing, **trace} if isinstance(existing, dict) else trace
|
||||||
|
return merged, trace
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_trace_after_policy(details: dict[str, Any], trace: dict[str, str]) -> dict[str, Any]:
|
||||||
|
if not trace:
|
||||||
|
return details
|
||||||
|
if details.get("_trace") == trace:
|
||||||
|
return details
|
||||||
|
restored = dict(details)
|
||||||
|
existing = restored.get("_trace")
|
||||||
|
restored["_trace"] = {**existing, **trace} if isinstance(existing, dict) else trace
|
||||||
|
return restored
|
||||||
|
|
||||||
|
|
||||||
def audit_event(
|
def audit_event(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -44,6 +146,8 @@ def audit_event(
|
|||||||
object_type: str | None = None,
|
object_type: str | None = None,
|
||||||
object_id: str | None = None,
|
object_id: str | None = None,
|
||||||
details: dict[str, Any] | None = None,
|
details: dict[str, Any] | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
causation_id: str | None = None,
|
||||||
commit: bool = False,
|
commit: bool = False,
|
||||||
) -> AuditLog:
|
) -> AuditLog:
|
||||||
"""Persist one audit event.
|
"""Persist one audit event.
|
||||||
@@ -56,6 +160,16 @@ def audit_event(
|
|||||||
if scope not in {"tenant", "system"}:
|
if scope not in {"tenant", "system"}:
|
||||||
raise ValueError(f"Unsupported audit scope: {scope}")
|
raise ValueError(f"Unsupported audit scope: {scope}")
|
||||||
|
|
||||||
|
traced_details, trace = _trace_details(
|
||||||
|
_sanitize_details(details or {}),
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
causation_id=causation_id,
|
||||||
|
)
|
||||||
|
stored_details = _restore_trace_after_policy(
|
||||||
|
sanitize_audit_details_for_policy(session, traced_details),
|
||||||
|
trace,
|
||||||
|
)
|
||||||
|
|
||||||
item = AuditLog(
|
item = AuditLog(
|
||||||
scope=scope,
|
scope=scope,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
@@ -64,13 +178,24 @@ def audit_event(
|
|||||||
action=action,
|
action=action,
|
||||||
object_type=object_type,
|
object_type=object_type,
|
||||||
object_id=object_id,
|
object_id=object_id,
|
||||||
details=sanitize_audit_details_for_policy(session, _sanitize_details(details or {})),
|
details=stored_details,
|
||||||
)
|
)
|
||||||
session.add(item)
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
record_change(
|
||||||
|
session,
|
||||||
|
module_id=AUDIT_MODULE_ID,
|
||||||
|
collection=AUDIT_SYSTEM_EVENTS_COLLECTION if scope == "system" else AUDIT_TENANT_EVENTS_COLLECTION,
|
||||||
|
resource_type="audit_log",
|
||||||
|
resource_id=item.id,
|
||||||
|
operation="created",
|
||||||
|
tenant_id=None if scope == "system" else tenant_id,
|
||||||
|
actor_type="user" if user_id else ("api_key" if api_key_id else None),
|
||||||
|
actor_id=user_id or api_key_id,
|
||||||
|
payload={"scope": scope, "action": action, "object_type": object_type, "object_id": object_id},
|
||||||
|
)
|
||||||
if commit:
|
if commit:
|
||||||
session.commit()
|
session.commit()
|
||||||
else:
|
|
||||||
session.flush()
|
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
@@ -83,6 +208,8 @@ def audit_from_principal(
|
|||||||
object_type: str | None = None,
|
object_type: str | None = None,
|
||||||
object_id: str | None = None,
|
object_id: str | None = None,
|
||||||
details: dict[str, Any] | None = None,
|
details: dict[str, Any] | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
causation_id: str | None = None,
|
||||||
commit: bool = False,
|
commit: bool = False,
|
||||||
) -> AuditLog:
|
) -> AuditLog:
|
||||||
return audit_event(
|
return audit_event(
|
||||||
@@ -95,5 +222,7 @@ def audit_from_principal(
|
|||||||
object_type=object_type,
|
object_type=object_type,
|
||||||
object_id=object_id,
|
object_id=object_id,
|
||||||
details=details,
|
details=details,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
causation_id=causation_id,
|
||||||
commit=commit,
|
commit=commit,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from govoplan_core.core.module_installer import (
|
|||||||
update_module_installer_request,
|
update_module_installer_request,
|
||||||
update_module_installer_daemon_status,
|
update_module_installer_daemon_status,
|
||||||
)
|
)
|
||||||
|
from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics
|
||||||
from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog
|
from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog
|
||||||
from govoplan_core.core.module_management import (
|
from govoplan_core.core.module_management import (
|
||||||
configured_enabled_modules,
|
configured_enabled_modules,
|
||||||
@@ -80,10 +81,69 @@ def main() -> int:
|
|||||||
parser.add_argument("--catalog-signing-key-id", help="Key id to record when signing a package catalog.")
|
parser.add_argument("--catalog-signing-key-id", help="Key id to record when signing a package catalog.")
|
||||||
parser.add_argument("--catalog-signing-private-key", type=Path, help="PEM Ed25519 private key used by --sign-package-catalog.")
|
parser.add_argument("--catalog-signing-private-key", type=Path, help="PEM Ed25519 private key used by --sign-package-catalog.")
|
||||||
parser.add_argument("--catalog-output", type=Path, help="Output path for --sign-package-catalog. Defaults to overwriting the input file.")
|
parser.add_argument("--catalog-output", type=Path, help="Output path for --sign-package-catalog. Defaults to overwriting the input file.")
|
||||||
|
parser.add_argument("--validate-license", nargs="?", const="", metavar="PATH", help="Validate a license JSON file, or the configured license when PATH is omitted.")
|
||||||
|
parser.add_argument("--require-trusted-license", action="store_true", help="Require license validation to trust an Ed25519 signature.")
|
||||||
|
parser.add_argument("--license-trusted-key", action="append", default=[], metavar="KEY_ID=BASE64_PUBLIC_KEY", help="Trusted Ed25519 license public key; may be repeated.")
|
||||||
|
parser.add_argument("--license-required-feature", action="append", default=[], help="License feature to check; may be repeated.")
|
||||||
|
parser.add_argument("--issue-license", type=Path, metavar="PATH", help="Write a signed offline license JSON file.")
|
||||||
|
parser.add_argument("--license-id", help="License id for --issue-license.")
|
||||||
|
parser.add_argument("--license-subject", help="License subject/customer for --issue-license.")
|
||||||
|
parser.add_argument("--license-feature", action="append", default=[], help="Feature entitlement to include in --issue-license; may be repeated.")
|
||||||
|
parser.add_argument("--license-valid-from", help="License valid_from timestamp. Defaults to now.")
|
||||||
|
parser.add_argument("--license-valid-until", help="License valid_until timestamp.")
|
||||||
|
parser.add_argument("--license-signing-key-id", help="Signing key id for --issue-license.")
|
||||||
|
parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.")
|
||||||
|
parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.")
|
||||||
|
parser.add_argument("--license-notes", help="Optional operator note for --issue-license.")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url)
|
runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url)
|
||||||
try:
|
try:
|
||||||
|
if args.issue_license:
|
||||||
|
if not args.license_id or not args.license_subject or not args.license_valid_until:
|
||||||
|
raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.")
|
||||||
|
if not args.license_signing_key_id or not args.license_signing_private_key:
|
||||||
|
raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.")
|
||||||
|
if not [item for item in args.license_feature if item.strip()]:
|
||||||
|
raise ModuleInstallerError("--issue-license requires at least one --license-feature.")
|
||||||
|
try:
|
||||||
|
path = issue_module_license(
|
||||||
|
path=args.issue_license,
|
||||||
|
license_id=args.license_id,
|
||||||
|
subject=args.license_subject,
|
||||||
|
features=args.license_feature,
|
||||||
|
valid_from=args.license_valid_from,
|
||||||
|
valid_until=args.license_valid_until,
|
||||||
|
signing_key_id=args.license_signing_key_id,
|
||||||
|
signing_private_key_path=args.license_signing_private_key,
|
||||||
|
issuer=args.license_issuer,
|
||||||
|
notes=args.license_notes,
|
||||||
|
)
|
||||||
|
except (OSError, ValueError) as exc:
|
||||||
|
raise ModuleInstallerError(str(exc)) from exc
|
||||||
|
_print_result(
|
||||||
|
{
|
||||||
|
"issued": True,
|
||||||
|
"path": str(path),
|
||||||
|
"license_id": args.license_id,
|
||||||
|
"subject": args.license_subject,
|
||||||
|
"features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())),
|
||||||
|
"valid_until": args.license_valid_until,
|
||||||
|
"key_id": args.license_signing_key_id,
|
||||||
|
},
|
||||||
|
output_format=args.format,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
if args.validate_license is not None:
|
||||||
|
path = Path(args.validate_license).expanduser() if args.validate_license else None
|
||||||
|
result = module_license_diagnostics(
|
||||||
|
path,
|
||||||
|
require_trusted=args.require_trusted_license,
|
||||||
|
trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"),
|
||||||
|
required_features=args.license_required_feature,
|
||||||
|
)
|
||||||
|
_print_result(result, output_format=args.format)
|
||||||
|
return 0 if result.get("valid") and not result.get("missing_features") else 1
|
||||||
if args.sign_package_catalog:
|
if args.sign_package_catalog:
|
||||||
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
if not args.catalog_signing_key_id or not args.catalog_signing_private_key:
|
||||||
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.")
|
||||||
@@ -101,7 +161,7 @@ def main() -> int:
|
|||||||
path,
|
path,
|
||||||
require_trusted=args.require_signed_catalog,
|
require_trusted=args.require_signed_catalog,
|
||||||
approved_channels=tuple(args.approved_catalog_channel),
|
approved_channels=tuple(args.approved_catalog_channel),
|
||||||
trusted_keys=_trusted_catalog_keys_from_args(args.catalog_trusted_key),
|
trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"),
|
||||||
)
|
)
|
||||||
_print_result(result, output_format=args.format)
|
_print_result(result, output_format=args.format)
|
||||||
return 0 if result.get("valid") else 1
|
return 0 if result.get("valid") else 1
|
||||||
@@ -322,6 +382,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru
|
|||||||
health_urls=_list_option(options, "health_urls") or _list_option(options, "health_url") or ([args.health_url] if args.health_url else []),
|
health_urls=_list_option(options, "health_urls") or _list_option(options, "health_url") or ([args.health_url] if args.health_url else []),
|
||||||
health_timeout_seconds=_float_option(options, "health_timeout_seconds", args.health_timeout_seconds),
|
health_timeout_seconds=_float_option(options, "health_timeout_seconds", args.health_timeout_seconds),
|
||||||
health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds),
|
health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds),
|
||||||
|
request_context=_request_context(request),
|
||||||
)
|
)
|
||||||
update_module_installer_request(
|
update_module_installer_request(
|
||||||
runtime_dir=runtime_dir,
|
runtime_dir=runtime_dir,
|
||||||
@@ -362,6 +423,18 @@ def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _request_context(request: dict[str, object]) -> dict[str, object]:
|
||||||
|
context: dict[str, object] = {
|
||||||
|
"request_id": request.get("request_id"),
|
||||||
|
"requested_by": request.get("requested_by"),
|
||||||
|
}
|
||||||
|
if request.get("retry_of"):
|
||||||
|
context["retry_of"] = request["retry_of"]
|
||||||
|
if isinstance(request.get("trace"), dict):
|
||||||
|
context["trace"] = request["trace"]
|
||||||
|
return {key: value for key, value in context.items() if value is not None}
|
||||||
|
|
||||||
|
|
||||||
def _str_option(options: object, key: str) -> str | None:
|
def _str_option(options: object, key: str) -> str | None:
|
||||||
if not isinstance(options, dict):
|
if not isinstance(options, dict):
|
||||||
return None
|
return None
|
||||||
@@ -403,14 +476,14 @@ def _list_option(options: object, key: str) -> list[str]:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def _trusted_catalog_keys_from_args(values: list[str]) -> dict[str, str] | None:
|
def _trusted_keys_from_args(values: list[str], *, flag_name: str) -> dict[str, str] | None:
|
||||||
if not values:
|
if not values:
|
||||||
return None
|
return None
|
||||||
keys: dict[str, str] = {}
|
keys: dict[str, str] = {}
|
||||||
for value in values:
|
for value in values:
|
||||||
key_id, separator, public_key = value.partition("=")
|
key_id, separator, public_key = value.partition("=")
|
||||||
if not separator or not key_id.strip() or not public_key.strip():
|
if not separator or not key_id.strip() or not public_key.strip():
|
||||||
raise ModuleInstallerError("--catalog-trusted-key must use KEY_ID=BASE64_PUBLIC_KEY.")
|
raise ModuleInstallerError(f"{flag_name} must use KEY_ID=BASE64_PUBLIC_KEY.")
|
||||||
keys[key_id.strip()] = public_key.strip()
|
keys[key_id.strip()] = public_key.strip()
|
||||||
return keys
|
return keys
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Literal, Protocol, runtime_checkable
|
from typing import Literal, Protocol, cast, runtime_checkable
|
||||||
|
|
||||||
from govoplan_core.core.modules import AccessDecision
|
from govoplan_core.core.modules import AccessDecision
|
||||||
|
|
||||||
@@ -17,11 +17,12 @@ CAPABILITY_ACCESS_PRINCIPAL_RESOLVER = "access.principalResolver"
|
|||||||
CAPABILITY_ACCESS_DIRECTORY = "access.directory"
|
CAPABILITY_ACCESS_DIRECTORY = "access.directory"
|
||||||
CAPABILITY_ACCESS_PERMISSION_EVALUATOR = "access.permissionEvaluator"
|
CAPABILITY_ACCESS_PERMISSION_EVALUATOR = "access.permissionEvaluator"
|
||||||
CAPABILITY_ACCESS_RESOURCE_ACCESS = "access.resourceAccess"
|
CAPABILITY_ACCESS_RESOURCE_ACCESS = "access.resourceAccess"
|
||||||
|
CAPABILITY_ACCESS_SEMANTIC_DIRECTORY = "access.semanticDirectory"
|
||||||
|
CAPABILITY_ACCESS_EXPLANATION = "access.explanation"
|
||||||
CAPABILITY_ACCESS_TENANT_PROVISIONER = "access.tenantProvisioner"
|
CAPABILITY_ACCESS_TENANT_PROVISIONER = "access.tenantProvisioner"
|
||||||
CAPABILITY_ACCESS_ADMINISTRATION = "access.administration"
|
CAPABILITY_ACCESS_ADMINISTRATION = "access.administration"
|
||||||
CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER = "access.governanceMaterializer"
|
CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER = "access.governanceMaterializer"
|
||||||
CAPABILITY_TENANCY_TENANT_RESOLVER = "tenancy.tenantResolver"
|
CAPABILITY_TENANCY_TENANT_RESOLVER = "tenancy.tenantResolver"
|
||||||
CAPABILITY_SECURITY_SECRET_PROVIDER = "security.secretProvider"
|
|
||||||
CAPABILITY_AUDIT_SINK = "audit.sink"
|
CAPABILITY_AUDIT_SINK = "audit.sink"
|
||||||
|
|
||||||
ACCESS_CAPABILITY_NAMES = frozenset(
|
ACCESS_CAPABILITY_NAMES = frozenset(
|
||||||
@@ -30,20 +31,47 @@ ACCESS_CAPABILITY_NAMES = frozenset(
|
|||||||
CAPABILITY_ACCESS_DIRECTORY,
|
CAPABILITY_ACCESS_DIRECTORY,
|
||||||
CAPABILITY_ACCESS_PERMISSION_EVALUATOR,
|
CAPABILITY_ACCESS_PERMISSION_EVALUATOR,
|
||||||
CAPABILITY_ACCESS_RESOURCE_ACCESS,
|
CAPABILITY_ACCESS_RESOURCE_ACCESS,
|
||||||
|
CAPABILITY_ACCESS_SEMANTIC_DIRECTORY,
|
||||||
|
CAPABILITY_ACCESS_EXPLANATION,
|
||||||
CAPABILITY_ACCESS_TENANT_PROVISIONER,
|
CAPABILITY_ACCESS_TENANT_PROVISIONER,
|
||||||
CAPABILITY_ACCESS_ADMINISTRATION,
|
CAPABILITY_ACCESS_ADMINISTRATION,
|
||||||
CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER,
|
CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER,
|
||||||
CAPABILITY_TENANCY_TENANT_RESOLVER,
|
CAPABILITY_TENANCY_TENANT_RESOLVER,
|
||||||
CAPABILITY_SECURITY_SECRET_PROVIDER,
|
|
||||||
CAPABILITY_AUDIT_SINK,
|
CAPABILITY_AUDIT_SINK,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
AuthMethod = Literal["session", "api_key", "service_account"]
|
AuthMethod = Literal["session", "api_key", "service_account"]
|
||||||
AccessSubjectKind = Literal["account", "user", "group", "role", "tenant", "api_key", "service_account"]
|
AccessSubjectKind = Literal[
|
||||||
|
"identity",
|
||||||
|
"account",
|
||||||
|
"user",
|
||||||
|
"group",
|
||||||
|
"role",
|
||||||
|
"tenant",
|
||||||
|
"organization_unit",
|
||||||
|
"function",
|
||||||
|
"api_key",
|
||||||
|
"service_account",
|
||||||
|
]
|
||||||
AccessStatus = Literal["active", "inactive", "suspended"]
|
AccessStatus = Literal["active", "inactive", "suspended"]
|
||||||
PermissionLevel = Literal["system", "tenant"]
|
PermissionLevel = Literal["system", "tenant"]
|
||||||
AuditOutcome = Literal["success", "failure", "denied", "unknown"]
|
AuditOutcome = Literal["success", "failure", "denied", "unknown"]
|
||||||
|
FunctionAssignmentSource = Literal["direct", "delegated", "acting_for", "directory", "governance", "system"]
|
||||||
|
FunctionDelegationMode = Literal["delegate", "act_in_place"]
|
||||||
|
AccessProvenanceKind = Literal[
|
||||||
|
"identity",
|
||||||
|
"account",
|
||||||
|
"tenant_membership",
|
||||||
|
"organization_unit",
|
||||||
|
"function",
|
||||||
|
"group",
|
||||||
|
"role",
|
||||||
|
"right",
|
||||||
|
"delegation",
|
||||||
|
"policy",
|
||||||
|
"system_actor",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -51,15 +79,75 @@ class PrincipalRef:
|
|||||||
account_id: str
|
account_id: str
|
||||||
membership_id: str | None
|
membership_id: str | None
|
||||||
tenant_id: str | None
|
tenant_id: str | None
|
||||||
|
identity_id: str | None = None
|
||||||
scopes: frozenset[str] = field(default_factory=frozenset)
|
scopes: frozenset[str] = field(default_factory=frozenset)
|
||||||
group_ids: frozenset[str] = field(default_factory=frozenset)
|
group_ids: frozenset[str] = field(default_factory=frozenset)
|
||||||
|
role_ids: frozenset[str] = field(default_factory=frozenset)
|
||||||
|
function_assignment_ids: frozenset[str] = field(default_factory=frozenset)
|
||||||
|
delegation_ids: frozenset[str] = field(default_factory=frozenset)
|
||||||
auth_method: AuthMethod = "session"
|
auth_method: AuthMethod = "session"
|
||||||
api_key_id: str | None = None
|
api_key_id: str | None = None
|
||||||
session_id: str | None = None
|
session_id: str | None = None
|
||||||
service_account_id: str | None = None
|
service_account_id: str | None = None
|
||||||
|
acting_for_account_id: str | None = None
|
||||||
email: str | None = None
|
email: str | None = None
|
||||||
display_name: str | None = None
|
display_name: str | None = None
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"account_id": self.account_id,
|
||||||
|
"membership_id": self.membership_id,
|
||||||
|
"tenant_id": self.tenant_id,
|
||||||
|
"identity_id": self.identity_id,
|
||||||
|
"scopes": sorted(self.scopes),
|
||||||
|
"group_ids": sorted(self.group_ids),
|
||||||
|
"role_ids": sorted(self.role_ids),
|
||||||
|
"function_assignment_ids": sorted(self.function_assignment_ids),
|
||||||
|
"delegation_ids": sorted(self.delegation_ids),
|
||||||
|
"auth_method": self.auth_method,
|
||||||
|
"api_key_id": self.api_key_id,
|
||||||
|
"session_id": self.session_id,
|
||||||
|
"service_account_id": self.service_account_id,
|
||||||
|
"acting_for_account_id": self.acting_for_account_id,
|
||||||
|
"email": self.email,
|
||||||
|
"display_name": self.display_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, object]) -> PrincipalRef:
|
||||||
|
return cls(
|
||||||
|
account_id=str(value["account_id"]),
|
||||||
|
membership_id=_optional_str(value.get("membership_id")),
|
||||||
|
tenant_id=_optional_str(value.get("tenant_id")),
|
||||||
|
identity_id=_optional_str(value.get("identity_id")),
|
||||||
|
scopes=_string_frozenset(value.get("scopes")),
|
||||||
|
group_ids=_string_frozenset(value.get("group_ids")),
|
||||||
|
role_ids=_string_frozenset(value.get("role_ids")),
|
||||||
|
function_assignment_ids=_string_frozenset(value.get("function_assignment_ids")),
|
||||||
|
delegation_ids=_string_frozenset(value.get("delegation_ids")),
|
||||||
|
auth_method=cast(AuthMethod, str(value.get("auth_method") or "session")),
|
||||||
|
api_key_id=_optional_str(value.get("api_key_id")),
|
||||||
|
session_id=_optional_str(value.get("session_id")),
|
||||||
|
service_account_id=_optional_str(value.get("service_account_id")),
|
||||||
|
acting_for_account_id=_optional_str(value.get("acting_for_account_id")),
|
||||||
|
email=_optional_str(value.get("email")),
|
||||||
|
display_name=_optional_str(value.get("display_name")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_str(value: object | None) -> str | None:
|
||||||
|
return str(value) if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _string_frozenset(value: object | None) -> frozenset[str]:
|
||||||
|
if value is None:
|
||||||
|
return frozenset()
|
||||||
|
if isinstance(value, str):
|
||||||
|
return frozenset({value})
|
||||||
|
if isinstance(value, Iterable):
|
||||||
|
return frozenset(str(item) for item in value)
|
||||||
|
raise TypeError("PrincipalRef collection fields must be strings or iterables")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class AccountRef:
|
class AccountRef:
|
||||||
@@ -87,6 +175,15 @@ class UserRef:
|
|||||||
status: AccessStatus = "active"
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IdentityRef:
|
||||||
|
id: str
|
||||||
|
display_name: str | None = None
|
||||||
|
primary_account_id: str | None = None
|
||||||
|
account_ids: tuple[str, ...] = ()
|
||||||
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class GroupRef:
|
class GroupRef:
|
||||||
id: str
|
id: str
|
||||||
@@ -105,6 +202,79 @@ class RoleRef:
|
|||||||
protected: bool = False
|
protected: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OrganizationUnitRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
name: str
|
||||||
|
parent_id: str | None = None
|
||||||
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class FunctionRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
organization_unit_id: str
|
||||||
|
slug: str
|
||||||
|
name: str
|
||||||
|
role_ids: tuple[str, ...] = ()
|
||||||
|
delegable: bool = False
|
||||||
|
act_in_place_allowed: bool = False
|
||||||
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class FunctionAssignmentRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
account_id: str
|
||||||
|
function_id: str
|
||||||
|
organization_unit_id: str
|
||||||
|
identity_id: str | None = None
|
||||||
|
applies_to_subunits: bool = False
|
||||||
|
source: FunctionAssignmentSource = "direct"
|
||||||
|
delegated_from_assignment_id: str | None = None
|
||||||
|
acting_for_account_id: str | None = None
|
||||||
|
valid_from: datetime | None = None
|
||||||
|
valid_until: datetime | None = None
|
||||||
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class FunctionDelegationRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
function_assignment_id: str
|
||||||
|
delegator_account_id: str
|
||||||
|
delegate_account_id: str
|
||||||
|
mode: FunctionDelegationMode = "delegate"
|
||||||
|
reason: str | None = None
|
||||||
|
valid_from: datetime | None = None
|
||||||
|
valid_until: datetime | None = None
|
||||||
|
status: AccessStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AccessDecisionProvenance:
|
||||||
|
kind: AccessProvenanceKind
|
||||||
|
id: str | None = None
|
||||||
|
label: str | None = None
|
||||||
|
tenant_id: str | None = None
|
||||||
|
source: str | None = None
|
||||||
|
details: Mapping[str, object] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"kind": self.kind,
|
||||||
|
"id": self.id,
|
||||||
|
"label": self.label,
|
||||||
|
"tenant_id": self.tenant_id,
|
||||||
|
"source": self.source,
|
||||||
|
"details": dict(self.details),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class AccessSubjectRef:
|
class AccessSubjectRef:
|
||||||
kind: AccessSubjectKind
|
kind: AccessSubjectKind
|
||||||
@@ -143,6 +313,8 @@ class AuditEvent:
|
|||||||
resource_type: str | None = None
|
resource_type: str | None = None
|
||||||
resource_id: str | None = None
|
resource_id: str | None = None
|
||||||
occurred_at: datetime | None = None
|
occurred_at: datetime | None = None
|
||||||
|
correlation_id: str | None = None
|
||||||
|
causation_id: str | None = None
|
||||||
details: Mapping[str, object] = field(default_factory=dict)
|
details: Mapping[str, object] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@@ -191,6 +363,43 @@ class AccessDirectory(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class AccessSemanticDirectory(AccessDirectory, Protocol):
|
||||||
|
def get_identity(self, identity_id: str) -> IdentityRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def accounts_for_identity(self, identity_id: str) -> Sequence[AccountRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def organization_units_for_tenant(self, tenant_id: str) -> Sequence[OrganizationUnitRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_function(self, function_id: str) -> FunctionRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def functions_for_organization_unit(
|
||||||
|
self,
|
||||||
|
organization_unit_id: str,
|
||||||
|
*,
|
||||||
|
include_subunits: bool = False,
|
||||||
|
) -> Sequence[FunctionRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_function_assignment(self, assignment_id: str) -> FunctionAssignmentRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def function_assignments_for_account(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
) -> Sequence[FunctionAssignmentRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
class PermissionEvaluator(Protocol):
|
class PermissionEvaluator(Protocol):
|
||||||
def scopes_grant(self, scopes: Iterable[str], required_scope: str) -> bool:
|
def scopes_grant(self, scopes: Iterable[str], required_scope: str) -> bool:
|
||||||
@@ -209,6 +418,26 @@ class ResourceAccessService(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class AccessExplanationService(Protocol):
|
||||||
|
def explain_scope_provenance(
|
||||||
|
self,
|
||||||
|
principal: PrincipalRef,
|
||||||
|
required_scope: str,
|
||||||
|
) -> Sequence[AccessDecisionProvenance]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def explain_resource_provenance(
|
||||||
|
self,
|
||||||
|
principal: PrincipalRef,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
action: str,
|
||||||
|
) -> Sequence[AccessDecisionProvenance]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
class TenantAccessProvisioner(Protocol):
|
class TenantAccessProvisioner(Protocol):
|
||||||
def ensure_default_roles(self, session: object, tenant: object | None = None) -> Mapping[str, object]:
|
def ensure_default_roles(self, session: object, tenant: object | None = None) -> Mapping[str, object]:
|
||||||
@@ -254,18 +483,6 @@ class AccessGovernanceMaterializer(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
|
||||||
class SecretProvider(Protocol):
|
|
||||||
def store_secret(self, *, scope: str, name: str, value: str) -> str:
|
|
||||||
...
|
|
||||||
|
|
||||||
def read_secret(self, secret_ref: str) -> str | None:
|
|
||||||
...
|
|
||||||
|
|
||||||
def delete_secret(self, secret_ref: str) -> None:
|
|
||||||
...
|
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
class AuditSink(Protocol):
|
class AuditSink(Protocol):
|
||||||
def record(self, event: AuditEvent) -> None:
|
def record(self, event: AuditEvent) -> None:
|
||||||
|
|||||||
253
src/govoplan_core/core/change_sequence.py
Normal file
253
src/govoplan_core/core/change_sequence.py
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Iterable, Sequence
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, DateTime, Index, Integer, JSON, String, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||||
|
|
||||||
|
from govoplan_core.db.base import Base, utcnow
|
||||||
|
|
||||||
|
WATERMARK_PREFIX = "seq:"
|
||||||
|
|
||||||
|
|
||||||
|
class ChangeSequenceEntry(Base):
|
||||||
|
__tablename__ = "core_change_sequence"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_core_change_sequence_tenant_module_id", "tenant_id", "module_id", "id"),
|
||||||
|
Index("ix_core_change_sequence_collection_id", "collection", "id"),
|
||||||
|
Index("ix_core_change_sequence_resource", "module_id", "resource_type", "resource_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||||
|
tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||||
|
module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
|
||||||
|
collection: Mapped[str] = mapped_column(String(150), nullable=False, index=True)
|
||||||
|
resource_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
|
||||||
|
resource_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||||
|
operation: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
|
||||||
|
actor_type: Mapped[str | None] = mapped_column(String(30), nullable=True, index=True)
|
||||||
|
actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||||
|
payload: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False, index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ChangeSequenceRetentionFloor(Base):
|
||||||
|
__tablename__ = "core_change_sequence_retention_floor"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("tenant_key", "module_id", "collection", name="uq_core_change_sequence_retention_scope"),
|
||||||
|
Index("ix_core_change_sequence_retention_scope", "tenant_key", "module_id", "collection"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||||
|
tenant_key: Mapped[str] = mapped_column(String(36), nullable=False, default="")
|
||||||
|
module_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||||
|
collection: Mapped[str] = mapped_column(String(150), nullable=False)
|
||||||
|
min_valid_sequence: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), nullable=False, default=0)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_key(tenant_id: str | None) -> str:
|
||||||
|
return tenant_id or ""
|
||||||
|
|
||||||
|
|
||||||
|
def encode_sequence_watermark(sequence_id: int | None) -> str:
|
||||||
|
return f"{WATERMARK_PREFIX}{int(sequence_id or 0)}"
|
||||||
|
|
||||||
|
|
||||||
|
def decode_sequence_watermark(value: str | None) -> int:
|
||||||
|
if value is None or not value.strip():
|
||||||
|
return 0
|
||||||
|
raw = value.strip()
|
||||||
|
if raw.startswith(WATERMARK_PREFIX):
|
||||||
|
raw = raw[len(WATERMARK_PREFIX):]
|
||||||
|
try:
|
||||||
|
sequence_id = int(raw)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ValueError("Invalid delta watermark") from exc
|
||||||
|
if sequence_id < 0:
|
||||||
|
raise ValueError("Invalid delta watermark")
|
||||||
|
return sequence_id
|
||||||
|
|
||||||
|
|
||||||
|
def record_change(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
module_id: str,
|
||||||
|
collection: str,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
operation: str,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
actor_type: str | None = None,
|
||||||
|
actor_id: str | None = None,
|
||||||
|
payload: dict[str, Any] | None = None,
|
||||||
|
) -> ChangeSequenceEntry:
|
||||||
|
entry = ChangeSequenceEntry(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
module_id=module_id,
|
||||||
|
collection=collection,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
operation=operation,
|
||||||
|
actor_type=actor_type,
|
||||||
|
actor_id=actor_id,
|
||||||
|
payload=payload or {},
|
||||||
|
)
|
||||||
|
session.add(entry)
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
def max_sequence_id(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Sequence[str] | None = None,
|
||||||
|
) -> int:
|
||||||
|
query = session.query(func.max(ChangeSequenceEntry.id))
|
||||||
|
if tenant_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id)
|
||||||
|
if module_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.module_id == module_id)
|
||||||
|
if collections:
|
||||||
|
query = query.filter(ChangeSequenceEntry.collection.in_(tuple(collections)))
|
||||||
|
return int(query.scalar() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
def min_sequence_id(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Sequence[str] | None = None,
|
||||||
|
) -> int:
|
||||||
|
query = session.query(func.min(ChangeSequenceEntry.id))
|
||||||
|
if tenant_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id)
|
||||||
|
if module_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.module_id == module_id)
|
||||||
|
if collections:
|
||||||
|
query = query.filter(ChangeSequenceEntry.collection.in_(tuple(collections)))
|
||||||
|
return int(query.scalar() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
def sequence_watermark_is_expired(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
since: int,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Sequence[str] | None = None,
|
||||||
|
) -> bool:
|
||||||
|
floor = retained_sequence_floor(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
module_id=module_id,
|
||||||
|
collections=collections,
|
||||||
|
)
|
||||||
|
return floor > 0 and since < floor
|
||||||
|
|
||||||
|
|
||||||
|
def retained_sequence_floor(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Sequence[str] | None = None,
|
||||||
|
) -> int:
|
||||||
|
query = session.query(func.max(ChangeSequenceRetentionFloor.min_valid_sequence))
|
||||||
|
query = query.filter(ChangeSequenceRetentionFloor.tenant_key == _tenant_key(tenant_id))
|
||||||
|
if module_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceRetentionFloor.module_id == module_id)
|
||||||
|
if collections:
|
||||||
|
query = query.filter(ChangeSequenceRetentionFloor.collection.in_(tuple(collections)))
|
||||||
|
return int(query.scalar() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
def prune_sequence_entries(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
before_or_equal_id: int,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Sequence[str] | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Delete old sequence rows and record the new minimum safe watermark.
|
||||||
|
|
||||||
|
After pruning rows up to `before_or_equal_id`, clients with watermarks below
|
||||||
|
that value cannot be replayed safely and should receive a full snapshot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
cutoff = int(before_or_equal_id)
|
||||||
|
if cutoff <= 0:
|
||||||
|
return 0
|
||||||
|
query = session.query(ChangeSequenceEntry).filter(ChangeSequenceEntry.id <= cutoff)
|
||||||
|
if tenant_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id)
|
||||||
|
if module_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.module_id == module_id)
|
||||||
|
collection_values = tuple(collections or ())
|
||||||
|
if collection_values:
|
||||||
|
query = query.filter(ChangeSequenceEntry.collection.in_(collection_values))
|
||||||
|
scopes = {
|
||||||
|
(_tenant_key(entry.tenant_id), entry.module_id, entry.collection)
|
||||||
|
for entry in query.with_entities(
|
||||||
|
ChangeSequenceEntry.tenant_id,
|
||||||
|
ChangeSequenceEntry.module_id,
|
||||||
|
ChangeSequenceEntry.collection,
|
||||||
|
).distinct()
|
||||||
|
}
|
||||||
|
deleted = query.delete(synchronize_session=False)
|
||||||
|
for tenant_key, scoped_module_id, collection in scopes:
|
||||||
|
floor = session.query(ChangeSequenceRetentionFloor).filter(
|
||||||
|
ChangeSequenceRetentionFloor.tenant_key == tenant_key,
|
||||||
|
ChangeSequenceRetentionFloor.module_id == scoped_module_id,
|
||||||
|
ChangeSequenceRetentionFloor.collection == collection,
|
||||||
|
).one_or_none()
|
||||||
|
if floor is None:
|
||||||
|
floor = ChangeSequenceRetentionFloor(
|
||||||
|
tenant_key=tenant_key,
|
||||||
|
module_id=scoped_module_id,
|
||||||
|
collection=collection,
|
||||||
|
min_valid_sequence=cutoff,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
floor.min_valid_sequence = max(int(floor.min_valid_sequence or 0), cutoff)
|
||||||
|
session.add(floor)
|
||||||
|
return int(deleted)
|
||||||
|
|
||||||
|
|
||||||
|
def sequence_entries_since(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
since: int,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
module_id: str | None = None,
|
||||||
|
collections: Iterable[str] | None = None,
|
||||||
|
limit: int = 500,
|
||||||
|
) -> list[ChangeSequenceEntry]:
|
||||||
|
query = session.query(ChangeSequenceEntry).filter(ChangeSequenceEntry.id > since)
|
||||||
|
if tenant_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id)
|
||||||
|
if module_id is not None:
|
||||||
|
query = query.filter(ChangeSequenceEntry.module_id == module_id)
|
||||||
|
collection_values = tuple(collections or ())
|
||||||
|
if collection_values:
|
||||||
|
query = query.filter(ChangeSequenceEntry.collection.in_(collection_values))
|
||||||
|
return query.order_by(ChangeSequenceEntry.id.asc()).limit(max(1, limit)).all()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ChangeSequenceEntry",
|
||||||
|
"ChangeSequenceRetentionFloor",
|
||||||
|
"decode_sequence_watermark",
|
||||||
|
"encode_sequence_watermark",
|
||||||
|
"max_sequence_id",
|
||||||
|
"min_sequence_id",
|
||||||
|
"prune_sequence_entries",
|
||||||
|
"record_change",
|
||||||
|
"retained_sequence_floor",
|
||||||
|
"sequence_entries_since",
|
||||||
|
"sequence_watermark_is_expired",
|
||||||
|
]
|
||||||
418
src/govoplan_core/core/configuration_control.py
Normal file
418
src/govoplan_core/core/configuration_control.py
Normal file
@@ -0,0 +1,418 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.admin.settings import get_system_settings
|
||||||
|
from govoplan_core.core.change_sequence import record_change
|
||||||
|
from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change
|
||||||
|
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||||
|
from govoplan_core.security.time import utc_now
|
||||||
|
|
||||||
|
|
||||||
|
CONFIGURATION_CONTROL_SETTINGS_KEY = "_configuration_control"
|
||||||
|
CONFIGURATION_CONTROL_MODULE_ID = "core"
|
||||||
|
CONFIGURATION_CHANGES_COLLECTION = "core.configuration_changes"
|
||||||
|
CONFIGURATION_CHANGE_REQUEST_RESOURCE = "configuration_change_request"
|
||||||
|
CONFIGURATION_CHANGE_RECORD_RESOURCE = "configuration_change_record"
|
||||||
|
MAX_CONFIGURATION_CHANGE_HISTORY = 200
|
||||||
|
MAX_CONFIGURATION_CHANGE_REQUESTS = 200
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigurationControlError(Exception):
|
||||||
|
def __init__(self, message: str, *, code: str = "configuration_change_blocked", plan: dict[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
self.plan = plan or {}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationChangeApproval:
|
||||||
|
request: dict[str, Any] | None
|
||||||
|
plan: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_value_digest(value: object) -> str:
|
||||||
|
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_control_snapshot(session: Session) -> dict[str, list[dict[str, Any]]]:
|
||||||
|
item = get_system_settings(session)
|
||||||
|
control = _control_payload(item.settings or {})
|
||||||
|
return {
|
||||||
|
"requests": list(control.get("requests", [])),
|
||||||
|
"history": list(control.get("history", [])),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def create_configuration_change_request(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
key: str,
|
||||||
|
value: object,
|
||||||
|
actor_user_id: str,
|
||||||
|
actor_scopes: tuple[str, ...] | list[str],
|
||||||
|
dry_run: bool,
|
||||||
|
target: dict[str, Any] | None = None,
|
||||||
|
reason: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
field = classify_configuration_field(key)
|
||||||
|
if field is None:
|
||||||
|
raise ConfigurationControlError(
|
||||||
|
"This setting is not in the configuration safety catalog.",
|
||||||
|
code="unknown_configuration_field",
|
||||||
|
plan=plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict(),
|
||||||
|
)
|
||||||
|
if not field.ui_managed:
|
||||||
|
raise ConfigurationControlError(
|
||||||
|
"This setting is deployment-managed and cannot be edited through the UI.",
|
||||||
|
code="deployment_managed",
|
||||||
|
plan=plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict(),
|
||||||
|
)
|
||||||
|
|
||||||
|
item = get_system_settings(session)
|
||||||
|
settings = dict(item.settings or {})
|
||||||
|
control = _control_payload(settings)
|
||||||
|
now = _now()
|
||||||
|
request = {
|
||||||
|
"id": f"cfgreq-{uuid.uuid4().hex}",
|
||||||
|
"key": field.key,
|
||||||
|
"label": field.label,
|
||||||
|
"target": dict(target or {}),
|
||||||
|
"value_hash": configuration_value_digest(value),
|
||||||
|
"value_preview": _sanitize_value(field.key, value),
|
||||||
|
"dry_run": bool(dry_run),
|
||||||
|
"requested_by": actor_user_id,
|
||||||
|
"requested_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
"reason": reason.strip() if isinstance(reason, str) and reason.strip() else None,
|
||||||
|
"status": "pending_approval" if field.two_person_approval_required else "approved",
|
||||||
|
"approvals": [
|
||||||
|
{
|
||||||
|
"user_id": actor_user_id,
|
||||||
|
"approved_at": now,
|
||||||
|
"role": "requester",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"plan": plan_configuration_change(
|
||||||
|
field.key,
|
||||||
|
actor_scopes=actor_scopes,
|
||||||
|
value=value,
|
||||||
|
dry_run=bool(dry_run),
|
||||||
|
maintenance_mode=saved_maintenance_mode(session).enabled,
|
||||||
|
approval_count=1,
|
||||||
|
include_env_only=False,
|
||||||
|
).to_dict(),
|
||||||
|
}
|
||||||
|
control["requests"] = _trim_requests([request, *list(control.get("requests", []))])
|
||||||
|
settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control
|
||||||
|
item.settings = settings
|
||||||
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
_record_configuration_control_change(
|
||||||
|
session,
|
||||||
|
resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE,
|
||||||
|
resource_id=str(request["id"]),
|
||||||
|
operation="created",
|
||||||
|
actor_user_id=actor_user_id,
|
||||||
|
payload={"key": field.key, "status": request["status"]},
|
||||||
|
)
|
||||||
|
return dict(request)
|
||||||
|
|
||||||
|
|
||||||
|
def approve_configuration_change_request(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
request_id: str,
|
||||||
|
actor_user_id: str,
|
||||||
|
actor_scopes: tuple[str, ...] | list[str],
|
||||||
|
reason: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
item = get_system_settings(session)
|
||||||
|
settings = dict(item.settings or {})
|
||||||
|
control = _control_payload(settings)
|
||||||
|
requests = list(control.get("requests", []))
|
||||||
|
index, request = _find_request(requests, request_id)
|
||||||
|
if request is None:
|
||||||
|
raise ConfigurationControlError("Configuration change request not found.", code="configuration_request_not_found")
|
||||||
|
if request.get("status") in {"applied", "rejected"}:
|
||||||
|
raise ConfigurationControlError("Configuration change request is no longer approvable.", code="configuration_request_closed")
|
||||||
|
if request.get("requested_by") == actor_user_id:
|
||||||
|
raise ConfigurationControlError("The requester cannot be the second approver.", code="second_approver_required")
|
||||||
|
approvals = list(request.get("approvals", []))
|
||||||
|
if not any(item.get("user_id") == actor_user_id for item in approvals if isinstance(item, dict)):
|
||||||
|
approvals.append(
|
||||||
|
{
|
||||||
|
"user_id": actor_user_id,
|
||||||
|
"approved_at": _now(),
|
||||||
|
"role": "approver",
|
||||||
|
"reason": reason.strip() if isinstance(reason, str) and reason.strip() else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
request = dict(request)
|
||||||
|
request["approvals"] = approvals
|
||||||
|
request["updated_at"] = _now()
|
||||||
|
request["status"] = "approved" if _approval_count(request) >= 2 else "pending_approval"
|
||||||
|
request["plan"] = plan_configuration_change(
|
||||||
|
str(request.get("key") or ""),
|
||||||
|
actor_scopes=actor_scopes,
|
||||||
|
value=request.get("value_preview"),
|
||||||
|
dry_run=bool(request.get("dry_run")),
|
||||||
|
maintenance_mode=saved_maintenance_mode(session).enabled,
|
||||||
|
approval_count=_approval_count(request),
|
||||||
|
include_env_only=False,
|
||||||
|
).to_dict()
|
||||||
|
requests[index] = request
|
||||||
|
control["requests"] = _trim_requests(requests)
|
||||||
|
settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control
|
||||||
|
item.settings = settings
|
||||||
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
_record_configuration_control_change(
|
||||||
|
session,
|
||||||
|
resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE,
|
||||||
|
resource_id=str(request["id"]),
|
||||||
|
operation="updated",
|
||||||
|
actor_user_id=actor_user_id,
|
||||||
|
payload={"key": request.get("key"), "status": request.get("status")},
|
||||||
|
)
|
||||||
|
return dict(request)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_configuration_change_allowed(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
key: str,
|
||||||
|
value: object,
|
||||||
|
actor_user_id: str,
|
||||||
|
actor_scopes: tuple[str, ...] | list[str],
|
||||||
|
change_request_id: str | None = None,
|
||||||
|
target: dict[str, Any] | None = None,
|
||||||
|
dry_run: bool = False,
|
||||||
|
) -> ConfigurationChangeApproval:
|
||||||
|
field = classify_configuration_field(key)
|
||||||
|
if field is None:
|
||||||
|
plan = plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict()
|
||||||
|
raise ConfigurationControlError("This setting is not in the configuration safety catalog.", code="unknown_configuration_field", plan=plan)
|
||||||
|
|
||||||
|
request: dict[str, Any] | None = None
|
||||||
|
approval_count = 0
|
||||||
|
dry_run_satisfied = dry_run
|
||||||
|
requires_control = field.ui_managed and (field.dry_run_required or field.two_person_approval_required or field.maintenance_required)
|
||||||
|
if change_request_id:
|
||||||
|
request = _request_by_id(session, change_request_id)
|
||||||
|
if request is None:
|
||||||
|
raise ConfigurationControlError("Configuration change request not found.", code="configuration_request_not_found")
|
||||||
|
if request.get("status") in {"applied", "rejected"}:
|
||||||
|
raise ConfigurationControlError("Configuration change request is closed.", code="configuration_request_closed")
|
||||||
|
if request.get("key") != field.key:
|
||||||
|
raise ConfigurationControlError("Configuration change request does not match this setting.", code="configuration_request_mismatch")
|
||||||
|
if request.get("value_hash") != configuration_value_digest(value):
|
||||||
|
raise ConfigurationControlError("Configuration change request value does not match this apply payload.", code="configuration_value_mismatch")
|
||||||
|
approval_count = _approval_count(request)
|
||||||
|
dry_run_satisfied = bool(request.get("dry_run")) or dry_run_satisfied
|
||||||
|
elif requires_control:
|
||||||
|
plan = plan_configuration_change(
|
||||||
|
field.key,
|
||||||
|
actor_scopes=actor_scopes,
|
||||||
|
value=value,
|
||||||
|
dry_run=dry_run_satisfied,
|
||||||
|
maintenance_mode=saved_maintenance_mode(session).enabled,
|
||||||
|
approval_count=approval_count,
|
||||||
|
include_env_only=False,
|
||||||
|
).to_dict()
|
||||||
|
raise ConfigurationControlError(
|
||||||
|
"This configuration change requires a recorded change request.",
|
||||||
|
code="configuration_request_required",
|
||||||
|
plan=plan,
|
||||||
|
)
|
||||||
|
|
||||||
|
plan = plan_configuration_change(
|
||||||
|
field.key,
|
||||||
|
actor_scopes=actor_scopes,
|
||||||
|
value=value,
|
||||||
|
dry_run=dry_run_satisfied,
|
||||||
|
maintenance_mode=saved_maintenance_mode(session).enabled,
|
||||||
|
approval_count=approval_count,
|
||||||
|
include_env_only=False,
|
||||||
|
).to_dict()
|
||||||
|
if plan.get("blockers"):
|
||||||
|
raise ConfigurationControlError("Configuration change is blocked by safety guardrails.", plan=plan)
|
||||||
|
return ConfigurationChangeApproval(request=request, plan=plan)
|
||||||
|
|
||||||
|
|
||||||
|
def record_configuration_change_applied(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
key: str,
|
||||||
|
before_value: object,
|
||||||
|
after_value: object,
|
||||||
|
actor_user_id: str,
|
||||||
|
approval: ConfigurationChangeApproval | None = None,
|
||||||
|
target: dict[str, Any] | None = None,
|
||||||
|
audit_event: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
item = get_system_settings(session)
|
||||||
|
settings = dict(item.settings or {})
|
||||||
|
control = _control_payload(settings)
|
||||||
|
sequence = int(control.get("sequence") or 0) + 1
|
||||||
|
request = approval.request if approval else None
|
||||||
|
record = {
|
||||||
|
"id": f"cfgchg-{sequence:06d}-{uuid.uuid4().hex[:8]}",
|
||||||
|
"version": sequence,
|
||||||
|
"key": key,
|
||||||
|
"target": dict(target or {}),
|
||||||
|
"actor_user_id": actor_user_id,
|
||||||
|
"approval_request_id": request.get("id") if request else None,
|
||||||
|
"approval_user_ids": [item.get("user_id") for item in request.get("approvals", []) if isinstance(item, dict)] if request else [],
|
||||||
|
"audit_event": audit_event,
|
||||||
|
"before": _sanitize_value(key, before_value),
|
||||||
|
"after": _sanitize_value(key, after_value),
|
||||||
|
"rollback_value": _sanitize_value(key, before_value),
|
||||||
|
"plan": approval.plan if approval else plan_configuration_change(key, actor_scopes=(), value=after_value).to_dict(),
|
||||||
|
"created_at": _now(),
|
||||||
|
"status": "applied",
|
||||||
|
}
|
||||||
|
control["sequence"] = sequence
|
||||||
|
control["history"] = _trim_history([record, *list(control.get("history", []))])
|
||||||
|
if request:
|
||||||
|
requests = list(control.get("requests", []))
|
||||||
|
index, stored = _find_request(requests, str(request.get("id")))
|
||||||
|
if stored is not None:
|
||||||
|
stored = dict(stored)
|
||||||
|
stored["status"] = "applied"
|
||||||
|
stored["applied_change_id"] = record["id"]
|
||||||
|
stored["updated_at"] = _now()
|
||||||
|
requests[index] = stored
|
||||||
|
control["requests"] = _trim_requests(requests)
|
||||||
|
settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control
|
||||||
|
item.settings = settings
|
||||||
|
session.add(item)
|
||||||
|
session.flush()
|
||||||
|
_record_configuration_control_change(
|
||||||
|
session,
|
||||||
|
resource_type=CONFIGURATION_CHANGE_RECORD_RESOURCE,
|
||||||
|
resource_id=str(record["id"]),
|
||||||
|
operation="created",
|
||||||
|
actor_user_id=actor_user_id,
|
||||||
|
payload={"key": key, "status": record["status"], "version": record["version"]},
|
||||||
|
)
|
||||||
|
if request:
|
||||||
|
_record_configuration_control_change(
|
||||||
|
session,
|
||||||
|
resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE,
|
||||||
|
resource_id=str(request.get("id")),
|
||||||
|
operation="updated",
|
||||||
|
actor_user_id=actor_user_id,
|
||||||
|
payload={"key": key, "status": "applied", "applied_change_id": record["id"]},
|
||||||
|
)
|
||||||
|
return dict(record)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_configuration_control_change(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
resource_id: str,
|
||||||
|
operation: str,
|
||||||
|
actor_user_id: str,
|
||||||
|
payload: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
record_change(
|
||||||
|
session,
|
||||||
|
module_id=CONFIGURATION_CONTROL_MODULE_ID,
|
||||||
|
collection=CONFIGURATION_CHANGES_COLLECTION,
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_id=resource_id,
|
||||||
|
operation=operation,
|
||||||
|
actor_type="user",
|
||||||
|
actor_id=actor_user_id,
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_by_id(session: Session, request_id: str) -> dict[str, Any] | None:
|
||||||
|
item = get_system_settings(session)
|
||||||
|
_index, request = _find_request(list(_control_payload(item.settings or {}).get("requests", [])), request_id)
|
||||||
|
return dict(request) if request is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _find_request(requests: list[dict[str, Any]], request_id: str) -> tuple[int, dict[str, Any] | None]:
|
||||||
|
for index, request in enumerate(requests):
|
||||||
|
if request.get("id") == request_id:
|
||||||
|
return index, request
|
||||||
|
return -1, None
|
||||||
|
|
||||||
|
|
||||||
|
def _control_payload(settings: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
raw = settings.get(CONFIGURATION_CONTROL_SETTINGS_KEY)
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
return {"sequence": 0, "requests": [], "history": []}
|
||||||
|
return {
|
||||||
|
"sequence": int(raw.get("sequence") or 0),
|
||||||
|
"requests": [dict(item) for item in raw.get("requests", []) if isinstance(item, dict)],
|
||||||
|
"history": [dict(item) for item in raw.get("history", []) if isinstance(item, dict)],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _approval_count(request: dict[str, Any]) -> int:
|
||||||
|
user_ids = []
|
||||||
|
for item in request.get("approvals", []):
|
||||||
|
if isinstance(item, dict) and item.get("user_id"):
|
||||||
|
user_ids.append(str(item["user_id"]))
|
||||||
|
return len(set(user_ids))
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitize_value(key: str, value: object) -> object:
|
||||||
|
field = classify_configuration_field(key)
|
||||||
|
if field is not None and field.secret_handling in {"reference_only", "env_only"}:
|
||||||
|
return _redact_secrets(value)
|
||||||
|
return _redact_secrets(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_secrets(value: object) -> object:
|
||||||
|
if isinstance(value, dict):
|
||||||
|
result: dict[str, object] = {}
|
||||||
|
for key, item in value.items():
|
||||||
|
if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}:
|
||||||
|
result[str(key)] = "<redacted>"
|
||||||
|
else:
|
||||||
|
result[str(key)] = _redact_secrets(item)
|
||||||
|
return result
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [_redact_secrets(item) for item in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
return requests[:MAX_CONFIGURATION_CHANGE_REQUESTS]
|
||||||
|
|
||||||
|
|
||||||
|
def _trim_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
return history[:MAX_CONFIGURATION_CHANGE_HISTORY]
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return utc_now().isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConfigurationChangeApproval",
|
||||||
|
"ConfigurationControlError",
|
||||||
|
"CONFIGURATION_CHANGE_RECORD_RESOURCE",
|
||||||
|
"CONFIGURATION_CHANGE_REQUEST_RESOURCE",
|
||||||
|
"CONFIGURATION_CHANGES_COLLECTION",
|
||||||
|
"CONFIGURATION_CONTROL_MODULE_ID",
|
||||||
|
"approve_configuration_change_request",
|
||||||
|
"configuration_control_snapshot",
|
||||||
|
"configuration_value_digest",
|
||||||
|
"create_configuration_change_request",
|
||||||
|
"ensure_configuration_change_allowed",
|
||||||
|
"record_configuration_change_applied",
|
||||||
|
]
|
||||||
883
src/govoplan_core/core/configuration_packages.py
Normal file
883
src/govoplan_core/core/configuration_packages.py
Normal file
@@ -0,0 +1,883 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from typing import Any, Literal, Protocol, runtime_checkable
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from govoplan_core.core.module_package_catalog import (
|
||||||
|
_canonical_catalog_bytes,
|
||||||
|
_catalog_freshness_state,
|
||||||
|
_catalog_optional_text,
|
||||||
|
_catalog_sequence,
|
||||||
|
_catalog_signature_state,
|
||||||
|
_catalog_source_exists,
|
||||||
|
_catalog_source_type,
|
||||||
|
_is_http_url,
|
||||||
|
_load_private_key,
|
||||||
|
_parse_trusted_keys,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider"
|
||||||
|
|
||||||
|
DiagnosticSeverity = Literal["blocker", "warning", "info"]
|
||||||
|
PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationModuleRequirement:
|
||||||
|
module_id: str
|
||||||
|
version: str | None = None
|
||||||
|
optional: bool = False
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {"module_id": self.module_id, "version": self.version, "optional": self.optional}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationPackageFragment:
|
||||||
|
module_id: str
|
||||||
|
fragment_type: str
|
||||||
|
fragment_id: str | None = None
|
||||||
|
payload: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"module_id": self.module_id,
|
||||||
|
"fragment_type": self.fragment_type,
|
||||||
|
"fragment_id": self.fragment_id,
|
||||||
|
"payload": dict(self.payload),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationPackageManifest:
|
||||||
|
package_id: str
|
||||||
|
name: str
|
||||||
|
version: str
|
||||||
|
description: str | None = None
|
||||||
|
publisher: str | None = None
|
||||||
|
category: str | None = None
|
||||||
|
license: str | None = None
|
||||||
|
required_modules: tuple[ConfigurationModuleRequirement, ...] = ()
|
||||||
|
required_capabilities: tuple[str, ...] = ()
|
||||||
|
optional_modules: tuple[ConfigurationModuleRequirement, ...] = ()
|
||||||
|
fragments: tuple[ConfigurationPackageFragment, ...] = ()
|
||||||
|
data_requirements: tuple[dict[str, object], ...] = ()
|
||||||
|
tags: tuple[str, ...] = ()
|
||||||
|
artifact_ref: str | None = None
|
||||||
|
artifact_sha256: str | None = None
|
||||||
|
signature: Mapping[str, Any] | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageManifest":
|
||||||
|
return cls(
|
||||||
|
package_id=_required_str(value, "package_id"),
|
||||||
|
name=_required_str(value, "name"),
|
||||||
|
version=_required_str(value, "version"),
|
||||||
|
description=_optional_str(value, "description"),
|
||||||
|
publisher=_optional_str(value, "publisher"),
|
||||||
|
category=_optional_str(value, "category"),
|
||||||
|
license=_optional_str(value, "license"),
|
||||||
|
required_modules=tuple(_module_requirements(value.get("required_modules"), optional=False)),
|
||||||
|
required_capabilities=tuple(_string_list(value.get("required_capabilities"))),
|
||||||
|
optional_modules=tuple(_module_requirements(value.get("optional_modules"), optional=True)),
|
||||||
|
fragments=tuple(_fragments(value.get("fragments"))),
|
||||||
|
data_requirements=tuple(_object_list(value.get("data_requirements"), field_name="data_requirements")),
|
||||||
|
tags=tuple(_string_list(value.get("tags"))),
|
||||||
|
artifact_ref=_optional_str(value, "artifact_ref"),
|
||||||
|
artifact_sha256=_optional_str(value, "artifact_sha256"),
|
||||||
|
signature=value.get("signature") if isinstance(value.get("signature"), Mapping) else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
payload: dict[str, object] = {
|
||||||
|
"package_id": self.package_id,
|
||||||
|
"name": self.name,
|
||||||
|
"version": self.version,
|
||||||
|
"required_modules": [item.to_dict() for item in self.required_modules],
|
||||||
|
"required_capabilities": list(self.required_capabilities),
|
||||||
|
"optional_modules": [item.to_dict() for item in self.optional_modules],
|
||||||
|
"fragments": [item.to_dict() for item in self.fragments],
|
||||||
|
"data_requirements": [dict(item) for item in self.data_requirements],
|
||||||
|
"tags": list(self.tags),
|
||||||
|
}
|
||||||
|
for key, value in (
|
||||||
|
("description", self.description),
|
||||||
|
("publisher", self.publisher),
|
||||||
|
("category", self.category),
|
||||||
|
("license", self.license),
|
||||||
|
("artifact_ref", self.artifact_ref),
|
||||||
|
("artifact_sha256", self.artifact_sha256),
|
||||||
|
):
|
||||||
|
if value is not None:
|
||||||
|
payload[key] = value
|
||||||
|
if self.signature is not None:
|
||||||
|
payload["signature"] = dict(self.signature)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationDiagnostic:
|
||||||
|
severity: DiagnosticSeverity
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
module_id: str | None = None
|
||||||
|
object_ref: str | None = None
|
||||||
|
resolution: str | None = None
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"severity": self.severity,
|
||||||
|
"code": self.code,
|
||||||
|
"message": self.message,
|
||||||
|
"module_id": self.module_id,
|
||||||
|
"object_ref": self.object_ref,
|
||||||
|
"resolution": self.resolution,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationPlanItem:
|
||||||
|
action: PlanAction
|
||||||
|
module_id: str
|
||||||
|
fragment_type: str
|
||||||
|
fragment_id: str | None = None
|
||||||
|
summary: str | None = None
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"action": self.action,
|
||||||
|
"module_id": self.module_id,
|
||||||
|
"fragment_type": self.fragment_type,
|
||||||
|
"fragment_id": self.fragment_id,
|
||||||
|
"summary": self.summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationRequiredData:
|
||||||
|
key: str
|
||||||
|
label: str
|
||||||
|
data_type: str = "string"
|
||||||
|
required: bool = True
|
||||||
|
secret: bool = False
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationRequiredData":
|
||||||
|
return cls(
|
||||||
|
key=_required_str(value, "key"),
|
||||||
|
label=_optional_str(value, "label") or _required_str(value, "key"),
|
||||||
|
data_type=_optional_str(value, "data_type") or _optional_str(value, "type") or "string",
|
||||||
|
required=_bool(value.get("required"), default=True),
|
||||||
|
secret=_bool(value.get("secret"), default=False),
|
||||||
|
description=_optional_str(value, "description"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"key": self.key,
|
||||||
|
"label": self.label,
|
||||||
|
"data_type": self.data_type,
|
||||||
|
"required": self.required,
|
||||||
|
"secret": self.secret,
|
||||||
|
"description": self.description,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationProviderDescription:
|
||||||
|
module_id: str
|
||||||
|
fragment_types: tuple[str, ...] = ()
|
||||||
|
schema_refs: Mapping[str, str] = field(default_factory=dict)
|
||||||
|
exported_scopes: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationPreflightContext:
|
||||||
|
tenant_id: str | None = None
|
||||||
|
operator_user_id: str | None = None
|
||||||
|
supplied_data: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
installed_modules: Mapping[str, str] = field(default_factory=dict)
|
||||||
|
capabilities: frozenset[str] = frozenset()
|
||||||
|
dry_run: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationPreflightResult:
|
||||||
|
diagnostics: tuple[ConfigurationDiagnostic, ...] = ()
|
||||||
|
required_data: tuple[ConfigurationRequiredData, ...] = ()
|
||||||
|
plan: tuple[ConfigurationPlanItem, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationApplyResult:
|
||||||
|
diagnostics: tuple[ConfigurationDiagnostic, ...] = ()
|
||||||
|
created_refs: Mapping[str, str] = field(default_factory=dict)
|
||||||
|
updated_refs: Mapping[str, str] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationExportSelection:
|
||||||
|
tenant_id: str | None = None
|
||||||
|
scopes: tuple[str, ...] = ()
|
||||||
|
module_ids: tuple[str, ...] = ()
|
||||||
|
object_refs: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationExportResult:
|
||||||
|
fragments: tuple[ConfigurationPackageFragment, ...] = ()
|
||||||
|
data_requirements: tuple[ConfigurationRequiredData, ...] = ()
|
||||||
|
diagnostics: tuple[ConfigurationDiagnostic, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class ConfigurationProvider(Protocol):
|
||||||
|
module_id: str
|
||||||
|
|
||||||
|
def describe(self) -> ConfigurationProviderDescription:
|
||||||
|
...
|
||||||
|
|
||||||
|
def preflight(self, fragment: ConfigurationPackageFragment, context: ConfigurationPreflightContext) -> ConfigurationPreflightResult:
|
||||||
|
...
|
||||||
|
|
||||||
|
def apply(self, fragment: ConfigurationPackageFragment, supplied_data: Mapping[str, Any], context: ConfigurationPreflightContext) -> ConfigurationApplyResult:
|
||||||
|
...
|
||||||
|
|
||||||
|
def export(self, selection: ConfigurationExportSelection, context: ConfigurationPreflightContext) -> ConfigurationExportResult:
|
||||||
|
...
|
||||||
|
|
||||||
|
def health(self, import_result: ConfigurationApplyResult, context: ConfigurationPreflightContext) -> tuple[ConfigurationDiagnostic, ...]:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
def dry_run_configuration_package(
|
||||||
|
package: ConfigurationPackageManifest | Mapping[str, Any],
|
||||||
|
providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider],
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationPreflightResult:
|
||||||
|
manifest = _configuration_package_manifest(package)
|
||||||
|
provider_map = _configuration_provider_map(providers)
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
required_data: list[ConfigurationRequiredData] = []
|
||||||
|
plan: list[ConfigurationPlanItem] = []
|
||||||
|
|
||||||
|
diagnostics.extend(_module_requirement_diagnostics(manifest, context))
|
||||||
|
diagnostics.extend(_capability_requirement_diagnostics(manifest, context))
|
||||||
|
for item in manifest.data_requirements:
|
||||||
|
requirement = ConfigurationRequiredData.from_mapping(item)
|
||||||
|
required_data.append(requirement)
|
||||||
|
if requirement.required and requirement.key not in context.supplied_data:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="required_data_missing",
|
||||||
|
message=f"Required operator data is missing: {requirement.label}.",
|
||||||
|
object_ref=requirement.key,
|
||||||
|
resolution="Collect this value in the configuration package wizard before applying.",
|
||||||
|
))
|
||||||
|
|
||||||
|
for fragment in manifest.fragments:
|
||||||
|
provider = provider_map.get(fragment.module_id)
|
||||||
|
if provider is None:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="configuration_provider_missing",
|
||||||
|
message=f"No configuration provider is registered for module {fragment.module_id!r}.",
|
||||||
|
module_id=fragment.module_id,
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Install and enable the module version that provides this configuration provider.",
|
||||||
|
))
|
||||||
|
plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider is missing."))
|
||||||
|
continue
|
||||||
|
description = provider.describe()
|
||||||
|
if description.fragment_types and fragment.fragment_type not in description.fragment_types:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="fragment_type_unsupported",
|
||||||
|
message=f"Provider {fragment.module_id!r} does not support fragment type {fragment.fragment_type!r}.",
|
||||||
|
module_id=fragment.module_id,
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Use a package version compatible with the installed module provider.",
|
||||||
|
))
|
||||||
|
plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Fragment type is unsupported."))
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result = provider.preflight(fragment, context)
|
||||||
|
except Exception as exc:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="provider_preflight_failed",
|
||||||
|
message=f"Configuration provider {fragment.module_id!r} preflight failed: {exc}",
|
||||||
|
module_id=fragment.module_id,
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Review provider diagnostics and package fragment payload.",
|
||||||
|
))
|
||||||
|
plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider preflight failed."))
|
||||||
|
continue
|
||||||
|
diagnostics.extend(result.diagnostics)
|
||||||
|
required_data.extend(result.required_data)
|
||||||
|
for requirement in result.required_data:
|
||||||
|
if requirement.required and requirement.key not in context.supplied_data:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="required_data_missing",
|
||||||
|
message=f"Required operator data is missing: {requirement.label}.",
|
||||||
|
module_id=fragment.module_id,
|
||||||
|
object_ref=requirement.key,
|
||||||
|
resolution="Collect this value in the configuration package wizard before applying.",
|
||||||
|
))
|
||||||
|
plan.extend(result.plan or (ConfigurationPlanItem(action="noop", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider reported no changes."),))
|
||||||
|
return ConfigurationPreflightResult(
|
||||||
|
diagnostics=tuple(_dedupe_diagnostics(diagnostics)),
|
||||||
|
required_data=tuple(_dedupe_required_data(required_data)),
|
||||||
|
plan=tuple(plan),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_configuration_package(
|
||||||
|
package: ConfigurationPackageManifest | Mapping[str, Any],
|
||||||
|
providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider],
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
supplied_data: Mapping[str, Any] | None = None,
|
||||||
|
) -> ConfigurationApplyResult:
|
||||||
|
manifest = _configuration_package_manifest(package)
|
||||||
|
apply_context = ConfigurationPreflightContext(
|
||||||
|
tenant_id=context.tenant_id,
|
||||||
|
operator_user_id=context.operator_user_id,
|
||||||
|
supplied_data=supplied_data if supplied_data is not None else context.supplied_data,
|
||||||
|
installed_modules=context.installed_modules,
|
||||||
|
capabilities=context.capabilities,
|
||||||
|
dry_run=False,
|
||||||
|
)
|
||||||
|
preflight = dry_run_configuration_package(manifest, providers, apply_context)
|
||||||
|
blockers = [item for item in preflight.diagnostics if item.severity == "blocker"]
|
||||||
|
if blockers:
|
||||||
|
return ConfigurationApplyResult(diagnostics=tuple(blockers))
|
||||||
|
provider_map = _configuration_provider_map(providers)
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = list(preflight.diagnostics)
|
||||||
|
created_refs: dict[str, str] = {}
|
||||||
|
updated_refs: dict[str, str] = {}
|
||||||
|
for fragment in manifest.fragments:
|
||||||
|
provider = provider_map[fragment.module_id]
|
||||||
|
try:
|
||||||
|
result = provider.apply(fragment, apply_context.supplied_data, apply_context)
|
||||||
|
diagnostics.extend(result.diagnostics)
|
||||||
|
created_refs.update(result.created_refs)
|
||||||
|
updated_refs.update(result.updated_refs)
|
||||||
|
diagnostics.extend(provider.health(result, apply_context))
|
||||||
|
except Exception as exc:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="provider_apply_failed",
|
||||||
|
message=f"Configuration provider {fragment.module_id!r} apply failed: {exc}",
|
||||||
|
module_id=fragment.module_id,
|
||||||
|
object_ref=fragment.fragment_id or fragment.fragment_type,
|
||||||
|
resolution="Stop the import, keep previous configuration, and inspect provider logs.",
|
||||||
|
))
|
||||||
|
return ConfigurationApplyResult(
|
||||||
|
diagnostics=tuple(_dedupe_diagnostics(diagnostics)),
|
||||||
|
created_refs=created_refs,
|
||||||
|
updated_refs=updated_refs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def export_configuration_package(
|
||||||
|
providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider],
|
||||||
|
selection: ConfigurationExportSelection,
|
||||||
|
context: ConfigurationPreflightContext,
|
||||||
|
) -> ConfigurationExportResult:
|
||||||
|
provider_map = _configuration_provider_map(providers)
|
||||||
|
module_ids = selection.module_ids or tuple(provider_map)
|
||||||
|
fragments: list[ConfigurationPackageFragment] = []
|
||||||
|
data_requirements: list[ConfigurationRequiredData] = []
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
for module_id in module_ids:
|
||||||
|
provider = provider_map.get(module_id)
|
||||||
|
if provider is None:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="warning",
|
||||||
|
code="configuration_provider_missing",
|
||||||
|
message=f"No configuration provider is registered for module {module_id!r}; skipping export.",
|
||||||
|
module_id=module_id,
|
||||||
|
resolution="Enable the module provider before exporting its configuration.",
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result = provider.export(selection, context)
|
||||||
|
except Exception as exc:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="provider_export_failed",
|
||||||
|
message=f"Configuration provider {module_id!r} export failed: {exc}",
|
||||||
|
module_id=module_id,
|
||||||
|
resolution="Review provider diagnostics before reusing this package export.",
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
fragments.extend(result.fragments)
|
||||||
|
data_requirements.extend(result.data_requirements)
|
||||||
|
diagnostics.extend(result.diagnostics)
|
||||||
|
return ConfigurationExportResult(
|
||||||
|
fragments=tuple(fragments),
|
||||||
|
data_requirements=tuple(_dedupe_required_data(data_requirements)),
|
||||||
|
diagnostics=tuple(_dedupe_diagnostics(diagnostics)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_package_catalog(
|
||||||
|
path: Path | str | None = None,
|
||||||
|
*,
|
||||||
|
require_trusted: bool | None = None,
|
||||||
|
approved_channels: tuple[str, ...] | None = None,
|
||||||
|
trusted_keys: dict[str, str] | None = None,
|
||||||
|
) -> tuple[dict[str, object], ...]:
|
||||||
|
validation = validate_configuration_package_catalog(
|
||||||
|
path,
|
||||||
|
require_trusted=require_trusted,
|
||||||
|
approved_channels=approved_channels,
|
||||||
|
trusted_keys=trusted_keys,
|
||||||
|
)
|
||||||
|
if not validation["valid"]:
|
||||||
|
raise ValueError(str(validation["error"] or "Configuration package catalog is invalid."))
|
||||||
|
return tuple(item for item in validation["packages"] if isinstance(item, dict))
|
||||||
|
|
||||||
|
|
||||||
|
def validate_configuration_package_catalog(
|
||||||
|
path: Path | str | None = None,
|
||||||
|
*,
|
||||||
|
require_trusted: bool | None = None,
|
||||||
|
approved_channels: tuple[str, ...] | None = None,
|
||||||
|
trusted_keys: dict[str, str] | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
source = _catalog_source(path)
|
||||||
|
configured = source is not None and _catalog_source_exists(source)
|
||||||
|
if source is not None and not _catalog_source_exists(source):
|
||||||
|
return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}")
|
||||||
|
read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() else None}
|
||||||
|
try:
|
||||||
|
payload, read_state = _read_catalog_payload_with_metadata(source)
|
||||||
|
packages = _normalize_catalog_packages(payload)
|
||||||
|
channel = _catalog_channel(payload)
|
||||||
|
sequence = _catalog_sequence(payload)
|
||||||
|
generated_at = _catalog_optional_text(payload, "generated_at")
|
||||||
|
not_before = _catalog_optional_text(payload, "not_before")
|
||||||
|
expires_at = _catalog_optional_text(payload, "expires_at")
|
||||||
|
effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted
|
||||||
|
effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels
|
||||||
|
effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys()
|
||||||
|
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
||||||
|
freshness = _catalog_freshness_state(payload)
|
||||||
|
replay = _catalog_replay_state(channel=channel, sequence=sequence)
|
||||||
|
except Exception as exc:
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc))
|
||||||
|
if signature_state.get("fatal"):
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"]))
|
||||||
|
if effective_approved_channels and channel not in effective_approved_channels:
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=f"Configuration package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.")
|
||||||
|
if effective_require_trusted and not signature_state["trusted"]:
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"] or "Configuration package catalog must be signed by a trusted key."))
|
||||||
|
if not freshness["valid"]:
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(freshness["error"]))
|
||||||
|
if not replay["valid"]:
|
||||||
|
return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(replay["error"]))
|
||||||
|
warnings = [str(item) for item in freshness.get("warnings", ()) if item]
|
||||||
|
warnings.extend(str(item) for item in replay.get("warnings", ()) if item)
|
||||||
|
if not signature_state["signed"]:
|
||||||
|
warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.")
|
||||||
|
elif not signature_state["trusted"]:
|
||||||
|
warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key."))
|
||||||
|
return _validation_result(
|
||||||
|
source,
|
||||||
|
valid=True,
|
||||||
|
configured=configured,
|
||||||
|
read_state=read_state,
|
||||||
|
packages=packages,
|
||||||
|
channel=channel,
|
||||||
|
sequence=sequence,
|
||||||
|
generated_at=generated_at,
|
||||||
|
not_before=not_before,
|
||||||
|
expires_at=expires_at,
|
||||||
|
signature_state=signature_state,
|
||||||
|
warnings=warnings,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path:
|
||||||
|
payload = _read_catalog_payload(path)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError("Only object-style configuration package catalogs can be signed.")
|
||||||
|
signed_payload = dict(payload)
|
||||||
|
signed_payload.pop("signature", None)
|
||||||
|
signed_payload.pop("signatures", None)
|
||||||
|
private_key = _load_private_key(private_key_path)
|
||||||
|
signature = private_key.sign(_canonical_catalog_bytes(signed_payload))
|
||||||
|
signed_payload["signature"] = {
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": key_id,
|
||||||
|
"value": base64.b64encode(signature).decode("ascii"),
|
||||||
|
}
|
||||||
|
target = output_path or path
|
||||||
|
target.write_text(json.dumps(signed_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
def record_configuration_package_catalog_acceptance(validation: dict[str, object]) -> None:
|
||||||
|
state_path = _configured_sequence_state_path()
|
||||||
|
if state_path is None or validation.get("valid") is not True:
|
||||||
|
return
|
||||||
|
channel = validation.get("channel")
|
||||||
|
sequence = validation.get("sequence")
|
||||||
|
if not isinstance(channel, str) or not isinstance(sequence, int):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.exists() else {}
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
state = {}
|
||||||
|
if not isinstance(state, dict):
|
||||||
|
state = {}
|
||||||
|
channels = state.get("channels")
|
||||||
|
if not isinstance(channels, dict):
|
||||||
|
channels = {}
|
||||||
|
channel_state = channels.get(channel)
|
||||||
|
if not isinstance(channel_state, dict):
|
||||||
|
channel_state = {}
|
||||||
|
channel_state["last_sequence"] = max(int(channel_state.get("last_sequence") or 0), sequence)
|
||||||
|
channel_state["accepted_at"] = datetime.now(tz=UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
channel_state["key_id"] = validation.get("key_id")
|
||||||
|
channel_state["source"] = validation.get("source") or validation.get("path")
|
||||||
|
channels[channel] = channel_state
|
||||||
|
state["channels"] = channels
|
||||||
|
state_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_package_manifest(package: ConfigurationPackageManifest | Mapping[str, Any]) -> ConfigurationPackageManifest:
|
||||||
|
if isinstance(package, ConfigurationPackageManifest):
|
||||||
|
return package
|
||||||
|
return ConfigurationPackageManifest.from_mapping(package)
|
||||||
|
|
||||||
|
|
||||||
|
def _configuration_provider_map(providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider]) -> dict[str, ConfigurationProvider]:
|
||||||
|
if isinstance(providers, Mapping):
|
||||||
|
return {str(key): provider for key, provider in providers.items()}
|
||||||
|
return {provider.module_id: provider for provider in providers}
|
||||||
|
|
||||||
|
|
||||||
|
def _module_requirement_diagnostics(manifest: ConfigurationPackageManifest, context: ConfigurationPreflightContext) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
installed = context.installed_modules
|
||||||
|
for requirement in manifest.required_modules:
|
||||||
|
installed_version = installed.get(requirement.module_id)
|
||||||
|
if installed_version is None:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="required_module_missing",
|
||||||
|
message=f"Required module {requirement.module_id!r} is not installed.",
|
||||||
|
module_id=requirement.module_id,
|
||||||
|
resolution="Install and enable the required module before applying this configuration package.",
|
||||||
|
))
|
||||||
|
elif requirement.version and installed_version != requirement.version:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="required_module_version_mismatch",
|
||||||
|
message=f"Required module {requirement.module_id!r} needs version {requirement.version}, but {installed_version} is installed.",
|
||||||
|
module_id=requirement.module_id,
|
||||||
|
resolution="Install a compatible module version or use a matching configuration package.",
|
||||||
|
))
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _capability_requirement_diagnostics(manifest: ConfigurationPackageManifest, context: ConfigurationPreflightContext) -> list[ConfigurationDiagnostic]:
|
||||||
|
diagnostics: list[ConfigurationDiagnostic] = []
|
||||||
|
capabilities = set(context.capabilities)
|
||||||
|
for capability in manifest.required_capabilities:
|
||||||
|
if capability not in capabilities:
|
||||||
|
diagnostics.append(ConfigurationDiagnostic(
|
||||||
|
severity="blocker",
|
||||||
|
code="required_capability_missing",
|
||||||
|
message=f"Required capability {capability!r} is not available.",
|
||||||
|
object_ref=capability,
|
||||||
|
resolution="Enable the module or platform capability before applying this configuration package.",
|
||||||
|
))
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe_diagnostics(items: Sequence[ConfigurationDiagnostic]) -> list[ConfigurationDiagnostic]:
|
||||||
|
seen: set[tuple[object, ...]] = set()
|
||||||
|
result: list[ConfigurationDiagnostic] = []
|
||||||
|
for item in items:
|
||||||
|
key = (item.severity, item.code, item.module_id, item.object_ref, item.message)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
result.append(item)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe_required_data(items: Sequence[ConfigurationRequiredData]) -> list[ConfigurationRequiredData]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[ConfigurationRequiredData] = []
|
||||||
|
for item in items:
|
||||||
|
if item.key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(item.key)
|
||||||
|
result.append(item)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_source(path: Path | str | None) -> Path | str | None:
|
||||||
|
if path is not None:
|
||||||
|
return path if isinstance(path, str) and _is_http_url(path) else Path(path).expanduser()
|
||||||
|
url = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_URL", "").strip()
|
||||||
|
if url:
|
||||||
|
return url
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG", "").strip()
|
||||||
|
return Path(value).expanduser() if value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_catalog_cache_path() -> Path | None:
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_CACHE", "").strip()
|
||||||
|
return Path(value).expanduser() if value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_sequence_state_path() -> Path | None:
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_SEQUENCE_STATE", "").strip()
|
||||||
|
return Path(value).expanduser() if value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_enforce_sequence() -> bool:
|
||||||
|
return os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_ENFORCE_SEQUENCE", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_require_signature() -> bool:
|
||||||
|
return os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_REQUIRE_SIGNATURE", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_approved_channels() -> tuple[str, ...]:
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_APPROVED_CHANNELS", "").strip()
|
||||||
|
return tuple(item.strip() for item in value.split(",") if item.strip()) if value else ()
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_trusted_keys() -> dict[str, str]:
|
||||||
|
file_value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "").strip()
|
||||||
|
if file_value:
|
||||||
|
path = Path(file_value).expanduser()
|
||||||
|
if not path.exists():
|
||||||
|
raise ValueError(f"Trusted configuration catalog key file does not exist: {path}")
|
||||||
|
return _parse_trusted_keys(path.read_text(encoding="utf-8"))
|
||||||
|
url_value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_URL", "").strip()
|
||||||
|
if url_value:
|
||||||
|
return _parse_trusted_keys(_read_trusted_keys_url(url_value))
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS", "").strip()
|
||||||
|
return _parse_trusted_keys(value) if value else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_trusted_keys_cache_path() -> Path | None:
|
||||||
|
value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE", "").strip()
|
||||||
|
return Path(value).expanduser() if value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_trusted_keys_url(url: str) -> str:
|
||||||
|
if not _is_http_url(url):
|
||||||
|
raise ValueError("Trusted configuration catalog key URL must use http:// or https://.")
|
||||||
|
cache_path = _configured_trusted_keys_cache_path()
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=15) as response:
|
||||||
|
body = response.read().decode("utf-8")
|
||||||
|
if cache_path is not None:
|
||||||
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
|
return body
|
||||||
|
except (OSError, urllib.error.URLError):
|
||||||
|
if cache_path is not None and cache_path.exists():
|
||||||
|
return cache_path.read_text(encoding="utf-8")
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _read_catalog_payload(source: Path | str | None) -> object:
|
||||||
|
payload, _metadata = _read_catalog_payload_with_metadata(source)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[object, dict[str, object]]:
|
||||||
|
cache_path = _configured_catalog_cache_path()
|
||||||
|
metadata: dict[str, object] = {"cache_used": False, "cache_path": str(cache_path) if cache_path else None}
|
||||||
|
if source is None:
|
||||||
|
return {"packages": []}, metadata
|
||||||
|
if isinstance(source, str) and _is_http_url(source):
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(source, timeout=15) as response:
|
||||||
|
body = response.read().decode("utf-8")
|
||||||
|
if cache_path is not None:
|
||||||
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
|
return json.loads(body), metadata
|
||||||
|
except (OSError, urllib.error.URLError):
|
||||||
|
if cache_path is not None and cache_path.exists():
|
||||||
|
metadata["cache_used"] = True
|
||||||
|
return json.loads(cache_path.read_text(encoding="utf-8")), metadata
|
||||||
|
raise
|
||||||
|
return json.loads(Path(source).read_text(encoding="utf-8")), metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_catalog_packages(payload: object) -> tuple[dict[str, object], ...]:
|
||||||
|
raw_items = payload.get("packages") if isinstance(payload, dict) else payload
|
||||||
|
if not isinstance(raw_items, list):
|
||||||
|
raise ValueError("Configuration package catalog must be a list or an object with a 'packages' list.")
|
||||||
|
return tuple(ConfigurationPackageManifest.from_mapping(item).to_dict() for item in raw_items if isinstance(item, Mapping))
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_channel(payload: object) -> str | None:
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
|
value = payload.get("channel")
|
||||||
|
return str(value).strip() if value is not None and str(value).strip() else None
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_replay_state(*, channel: str | None, sequence: int | None) -> dict[str, object]:
|
||||||
|
state_path = _configured_sequence_state_path()
|
||||||
|
if state_path is None:
|
||||||
|
return {"valid": True, "warnings": ()}
|
||||||
|
if not channel:
|
||||||
|
return {"valid": False, "error": "Configuration package catalog sequence replay protection needs a channel."}
|
||||||
|
if sequence is None:
|
||||||
|
return {"valid": False, "error": "Configuration package catalog sequence replay protection needs a sequence."}
|
||||||
|
if not state_path.exists():
|
||||||
|
return {"valid": True, "warnings": ()}
|
||||||
|
try:
|
||||||
|
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return {"valid": False, "error": f"Configuration catalog sequence state file is not valid JSON: {state_path}"}
|
||||||
|
channels = state.get("channels") if isinstance(state, dict) else None
|
||||||
|
channel_state = channels.get(channel) if isinstance(channels, dict) else None
|
||||||
|
last_sequence = channel_state.get("last_sequence") if isinstance(channel_state, dict) else None
|
||||||
|
if last_sequence is None:
|
||||||
|
return {"valid": True, "warnings": ()}
|
||||||
|
last = int(last_sequence)
|
||||||
|
if sequence < last:
|
||||||
|
return {"valid": False, "error": f"Configuration package catalog sequence {sequence} is older than accepted sequence {last} for channel {channel!r}."}
|
||||||
|
if sequence == last and _configured_enforce_sequence():
|
||||||
|
return {"valid": False, "error": f"Configuration package catalog sequence {sequence} was already accepted for channel {channel!r}."}
|
||||||
|
return {"valid": True, "warnings": ()}
|
||||||
|
|
||||||
|
|
||||||
|
def _validation_result(
|
||||||
|
source: Path | str | None,
|
||||||
|
*,
|
||||||
|
valid: bool = False,
|
||||||
|
configured: bool = False,
|
||||||
|
read_state: Mapping[str, object] | None = None,
|
||||||
|
packages: Sequence[Mapping[str, object]] = (),
|
||||||
|
channel: str | None = None,
|
||||||
|
sequence: int | None = None,
|
||||||
|
generated_at: str | None = None,
|
||||||
|
not_before: str | None = None,
|
||||||
|
expires_at: str | None = None,
|
||||||
|
signature_state: Mapping[str, object] | None = None,
|
||||||
|
warnings: Sequence[str] = (),
|
||||||
|
error: str | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
state = signature_state or {"signed": False, "trusted": False, "key_id": None}
|
||||||
|
read_state = read_state or {}
|
||||||
|
return {
|
||||||
|
"valid": valid,
|
||||||
|
"configured": configured,
|
||||||
|
"path": str(source) if source is not None else None,
|
||||||
|
"source": str(source) if source is not None else None,
|
||||||
|
"source_type": _catalog_source_type(source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
|
"packages": [dict(item) for item in packages],
|
||||||
|
"channel": channel,
|
||||||
|
"sequence": sequence,
|
||||||
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
|
"expires_at": expires_at,
|
||||||
|
"signed": bool(state.get("signed")),
|
||||||
|
"trusted": bool(state.get("trusted")),
|
||||||
|
"key_id": state.get("key_id"),
|
||||||
|
"warnings": list(warnings),
|
||||||
|
"error": error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _module_requirements(value: object, *, optional: bool) -> list[ConfigurationModuleRequirement]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ValueError("Configuration package module requirements must be lists.")
|
||||||
|
result: list[ConfigurationModuleRequirement] = []
|
||||||
|
for item in value:
|
||||||
|
if isinstance(item, str):
|
||||||
|
result.append(ConfigurationModuleRequirement(module_id=item, optional=optional))
|
||||||
|
elif isinstance(item, Mapping):
|
||||||
|
result.append(ConfigurationModuleRequirement(module_id=_required_str(item, "module_id"), version=_optional_str(item, "version"), optional=optional))
|
||||||
|
else:
|
||||||
|
raise ValueError("Configuration package module requirements must be strings or objects.")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _fragments(value: object) -> list[ConfigurationPackageFragment]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ValueError("Configuration package fragments must be a list.")
|
||||||
|
result: list[ConfigurationPackageFragment] = []
|
||||||
|
for item in value:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
raise ValueError("Configuration package fragments must be objects.")
|
||||||
|
payload = item.get("payload")
|
||||||
|
result.append(ConfigurationPackageFragment(
|
||||||
|
module_id=_required_str(item, "module_id"),
|
||||||
|
fragment_type=_required_str(item, "fragment_type"),
|
||||||
|
fragment_id=_optional_str(item, "fragment_id"),
|
||||||
|
payload=payload if isinstance(payload, Mapping) else {},
|
||||||
|
))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _object_list(value: object, *, field_name: str) -> list[dict[str, object]]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ValueError(f"Configuration package {field_name} must be a list.")
|
||||||
|
return [dict(item) for item in value if isinstance(item, Mapping)]
|
||||||
|
|
||||||
|
|
||||||
|
def _string_list(value: object) -> list[str]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ValueError("Configuration package list fields must be lists.")
|
||||||
|
return [str(item).strip() for item in value if str(item).strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _bool(value: object, *, default: bool) -> bool:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
return str(value).strip().casefold() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def _required_str(value: Mapping[str, Any], key: str) -> str:
|
||||||
|
item = _optional_str(value, key)
|
||||||
|
if not item:
|
||||||
|
raise ValueError(f"Configuration package entry is missing {key!r}.")
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_str(value: Mapping[str, Any], key: str) -> str | None:
|
||||||
|
item = value.get(key)
|
||||||
|
if item is None:
|
||||||
|
return None
|
||||||
|
text = str(item).strip()
|
||||||
|
return text or None
|
||||||
395
src/govoplan_core/core/configuration_safety.py
Normal file
395
src/govoplan_core/core/configuration_safety.py
Normal file
@@ -0,0 +1,395 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from govoplan_core.security.permissions import scopes_grant
|
||||||
|
|
||||||
|
|
||||||
|
ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"]
|
||||||
|
ConfigurationRisk = Literal["low", "medium", "high", "destructive"]
|
||||||
|
SecretHandling = Literal["none", "reference_only", "env_only"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationFieldSafety:
|
||||||
|
key: str
|
||||||
|
label: str
|
||||||
|
owner_module: str
|
||||||
|
scope: ConfigurationScope
|
||||||
|
storage: str
|
||||||
|
ui_managed: bool
|
||||||
|
risk: ConfigurationRisk
|
||||||
|
secret_handling: SecretHandling = "none"
|
||||||
|
required_scopes: tuple[str, ...] = ()
|
||||||
|
dry_run_required: bool = False
|
||||||
|
validation_required: bool = True
|
||||||
|
policy_explanation_required: bool = False
|
||||||
|
audit_event: str | None = None
|
||||||
|
maintenance_required: bool = False
|
||||||
|
two_person_approval_required: bool = False
|
||||||
|
rollback_history_required: bool = False
|
||||||
|
notes: str | None = None
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"key": self.key,
|
||||||
|
"label": self.label,
|
||||||
|
"owner_module": self.owner_module,
|
||||||
|
"scope": self.scope,
|
||||||
|
"storage": self.storage,
|
||||||
|
"ui_managed": self.ui_managed,
|
||||||
|
"risk": self.risk,
|
||||||
|
"secret_handling": self.secret_handling,
|
||||||
|
"required_scopes": list(self.required_scopes),
|
||||||
|
"dry_run_required": self.dry_run_required,
|
||||||
|
"validation_required": self.validation_required,
|
||||||
|
"policy_explanation_required": self.policy_explanation_required,
|
||||||
|
"audit_event": self.audit_event,
|
||||||
|
"maintenance_required": self.maintenance_required,
|
||||||
|
"two_person_approval_required": self.two_person_approval_required,
|
||||||
|
"rollback_history_required": self.rollback_history_required,
|
||||||
|
"notes": self.notes,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConfigurationChangeSafetyPlan:
|
||||||
|
key: str
|
||||||
|
allowed: bool
|
||||||
|
field: ConfigurationFieldSafety | None
|
||||||
|
risk: ConfigurationRisk | None = None
|
||||||
|
missing_scopes: tuple[str, ...] = ()
|
||||||
|
dry_run_required: bool = False
|
||||||
|
dry_run_satisfied: bool = False
|
||||||
|
approval_required: bool = False
|
||||||
|
approval_satisfied: bool = False
|
||||||
|
maintenance_required: bool = False
|
||||||
|
maintenance_satisfied: bool = False
|
||||||
|
rollback_history_required: bool = False
|
||||||
|
secret_handling: SecretHandling = "none"
|
||||||
|
audit_event: str | None = None
|
||||||
|
policy_explanation: str | None = None
|
||||||
|
blockers: tuple[str, ...] = ()
|
||||||
|
warnings: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"key": self.key,
|
||||||
|
"allowed": self.allowed,
|
||||||
|
"field": self.field.to_dict() if self.field else None,
|
||||||
|
"risk": self.risk,
|
||||||
|
"missing_scopes": list(self.missing_scopes),
|
||||||
|
"dry_run_required": self.dry_run_required,
|
||||||
|
"dry_run_satisfied": self.dry_run_satisfied,
|
||||||
|
"approval_required": self.approval_required,
|
||||||
|
"approval_satisfied": self.approval_satisfied,
|
||||||
|
"maintenance_required": self.maintenance_required,
|
||||||
|
"maintenance_satisfied": self.maintenance_satisfied,
|
||||||
|
"rollback_history_required": self.rollback_history_required,
|
||||||
|
"secret_handling": self.secret_handling,
|
||||||
|
"audit_event": self.audit_event,
|
||||||
|
"policy_explanation": self.policy_explanation,
|
||||||
|
"blockers": list(self.blockers),
|
||||||
|
"warnings": list(self.warnings),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = (
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="module_management.desired_enabled",
|
||||||
|
label="Enabled modules",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="system_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
required_scopes=("system:settings:write",),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="module_management.updated",
|
||||||
|
maintenance_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Changing enabled modules must use module preflight and installer rollback records.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="module_management.install_plan",
|
||||||
|
label="Module package install plan",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="system_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="destructive",
|
||||||
|
required_scopes=("system:settings:write",),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="module_install.requested",
|
||||||
|
maintenance_required=True,
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Package mutation needs maintenance mode, dry-run preflight, approval, and run records.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="configuration_packages.apply",
|
||||||
|
label="Configuration package apply",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="configuration_packages",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
required_scopes=("system:settings:write",),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="configuration_package.applied",
|
||||||
|
maintenance_required=True,
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Configuration package apply needs a dry-run, maintenance mode, approval, and version history.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="maintenance_mode",
|
||||||
|
label="Maintenance mode",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="system_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
required_scopes=("system:settings:write",),
|
||||||
|
validation_required=True,
|
||||||
|
audit_event="system_settings.updated",
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Maintenance mode controls platform availability and gates dangerous operations.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="privacy_retention_policy",
|
||||||
|
label="Privacy retention policy",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="system_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
required_scopes=("system:settings:write", "admin:policies:write"),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="privacy_retention.policy_updated",
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Retention reductions and destructive cleanup require explainable policy decisions and dry-run counts.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="governance_templates",
|
||||||
|
label="Governance templates",
|
||||||
|
owner_module="access",
|
||||||
|
scope="system",
|
||||||
|
storage="governance_templates",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
required_scopes=("system:governance:write",),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="governance_template.updated",
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Central groups, roles, and assignments need previews before materialization.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="mail_profiles.credentials",
|
||||||
|
label="Mail profile credentials",
|
||||||
|
owner_module="mail",
|
||||||
|
scope="tenant",
|
||||||
|
storage="module_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
secret_handling="reference_only",
|
||||||
|
required_scopes=("mail_servers:manage_credentials",),
|
||||||
|
validation_required=True,
|
||||||
|
audit_event="mail_server_profile.credential_updated",
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="UI must store secret references/placeholders only; secret values are never echoed.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="files.connector_profiles",
|
||||||
|
label="File connector profiles",
|
||||||
|
owner_module="files",
|
||||||
|
scope="tenant",
|
||||||
|
storage="module_settings",
|
||||||
|
ui_managed=True,
|
||||||
|
risk="high",
|
||||||
|
secret_handling="reference_only",
|
||||||
|
required_scopes=("files:file:admin",),
|
||||||
|
dry_run_required=True,
|
||||||
|
policy_explanation_required=True,
|
||||||
|
audit_event="files.connector_profile.updated",
|
||||||
|
two_person_approval_required=True,
|
||||||
|
rollback_history_required=True,
|
||||||
|
notes="Connector endpoints are UI-manageable, but passwords/tokens remain env or secret refs.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="DATABASE_URL",
|
||||||
|
label="Database URL",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="environment",
|
||||||
|
ui_managed=False,
|
||||||
|
risk="destructive",
|
||||||
|
secret_handling="env_only",
|
||||||
|
maintenance_required=True,
|
||||||
|
notes="Database connectivity remains deployment-managed and must not be changed from the running UI.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="MASTER_KEY_B64",
|
||||||
|
label="Master encryption key",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="environment",
|
||||||
|
ui_managed=False,
|
||||||
|
risk="destructive",
|
||||||
|
secret_handling="env_only",
|
||||||
|
maintenance_required=True,
|
||||||
|
two_person_approval_required=True,
|
||||||
|
notes="Encryption roots remain out of band; UI may only report missing/rotated state.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS",
|
||||||
|
label="Module package catalog trusted keys",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="environment",
|
||||||
|
ui_managed=False,
|
||||||
|
risk="high",
|
||||||
|
secret_handling="env_only",
|
||||||
|
notes="Trust roots are deployment-managed; UI can validate catalogs but should not edit key material.",
|
||||||
|
),
|
||||||
|
ConfigurationFieldSafety(
|
||||||
|
key="GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS",
|
||||||
|
label="Configuration package catalog trusted keys",
|
||||||
|
owner_module="core",
|
||||||
|
scope="system",
|
||||||
|
storage="environment",
|
||||||
|
ui_managed=False,
|
||||||
|
risk="high",
|
||||||
|
secret_handling="env_only",
|
||||||
|
notes="Configuration package trust roots are deployment-managed.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def configuration_safety_catalog(*, include_env_only: bool = True) -> tuple[ConfigurationFieldSafety, ...]:
|
||||||
|
if include_env_only:
|
||||||
|
return _CONFIGURATION_FIELD_SAFETY
|
||||||
|
return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.ui_managed)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_configuration_field(key: str) -> ConfigurationFieldSafety | None:
|
||||||
|
clean = key.strip()
|
||||||
|
for item in _CONFIGURATION_FIELD_SAFETY:
|
||||||
|
if item.key == clean:
|
||||||
|
return item
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def high_impact_configuration_fields() -> tuple[ConfigurationFieldSafety, ...]:
|
||||||
|
return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.risk in {"high", "destructive"})
|
||||||
|
|
||||||
|
|
||||||
|
def ui_managed_configuration_fields_requiring_approval() -> tuple[ConfigurationFieldSafety, ...]:
|
||||||
|
return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.ui_managed and item.two_person_approval_required)
|
||||||
|
|
||||||
|
|
||||||
|
def plan_configuration_change(
|
||||||
|
key: str,
|
||||||
|
*,
|
||||||
|
actor_scopes: tuple[str, ...] | list[str] = (),
|
||||||
|
value: object = None,
|
||||||
|
dry_run: bool = False,
|
||||||
|
maintenance_mode: bool = False,
|
||||||
|
approval_count: int = 0,
|
||||||
|
include_env_only: bool = True,
|
||||||
|
) -> ConfigurationChangeSafetyPlan:
|
||||||
|
field = classify_configuration_field(key)
|
||||||
|
if field is None:
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=key,
|
||||||
|
allowed=False,
|
||||||
|
field=None,
|
||||||
|
blockers=("unknown_configuration_field",),
|
||||||
|
policy_explanation="This setting is not in the configuration safety catalog.",
|
||||||
|
)
|
||||||
|
if not include_env_only and not field.ui_managed:
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=key,
|
||||||
|
allowed=False,
|
||||||
|
field=field,
|
||||||
|
risk=field.risk,
|
||||||
|
secret_handling=field.secret_handling,
|
||||||
|
blockers=("deployment_managed",),
|
||||||
|
policy_explanation="This setting is deployment-managed and cannot be edited through the UI.",
|
||||||
|
)
|
||||||
|
blockers: list[str] = []
|
||||||
|
warnings: list[str] = []
|
||||||
|
missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope))
|
||||||
|
if missing_scopes:
|
||||||
|
blockers.append("missing_required_scope")
|
||||||
|
if not field.ui_managed:
|
||||||
|
blockers.append("deployment_managed")
|
||||||
|
if field.dry_run_required and not dry_run:
|
||||||
|
blockers.append("dry_run_required")
|
||||||
|
if field.maintenance_required and not maintenance_mode:
|
||||||
|
blockers.append("maintenance_mode_required")
|
||||||
|
approval_required = field.two_person_approval_required
|
||||||
|
approval_satisfied = not approval_required or approval_count >= 2
|
||||||
|
if approval_required and not approval_satisfied:
|
||||||
|
blockers.append("two_person_approval_required")
|
||||||
|
if field.secret_handling == "reference_only" and _contains_plain_secret(value):
|
||||||
|
blockers.append("secret_reference_required")
|
||||||
|
if field.secret_handling == "env_only" and value is not None:
|
||||||
|
blockers.append("env_only_secret")
|
||||||
|
if field.rollback_history_required:
|
||||||
|
warnings.append("rollback_history_required")
|
||||||
|
return ConfigurationChangeSafetyPlan(
|
||||||
|
key=field.key,
|
||||||
|
allowed=not blockers,
|
||||||
|
field=field,
|
||||||
|
risk=field.risk,
|
||||||
|
missing_scopes=missing_scopes,
|
||||||
|
dry_run_required=field.dry_run_required,
|
||||||
|
dry_run_satisfied=not field.dry_run_required or dry_run,
|
||||||
|
approval_required=approval_required,
|
||||||
|
approval_satisfied=approval_satisfied,
|
||||||
|
maintenance_required=field.maintenance_required,
|
||||||
|
maintenance_satisfied=not field.maintenance_required or maintenance_mode,
|
||||||
|
rollback_history_required=field.rollback_history_required,
|
||||||
|
secret_handling=field.secret_handling,
|
||||||
|
audit_event=field.audit_event,
|
||||||
|
policy_explanation=_policy_explanation(field),
|
||||||
|
blockers=tuple(dict.fromkeys(blockers)),
|
||||||
|
warnings=tuple(dict.fromkeys(warnings)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_explanation(field: ConfigurationFieldSafety) -> str:
|
||||||
|
parts = [f"{field.label} is {field.risk}-risk"]
|
||||||
|
if field.dry_run_required:
|
||||||
|
parts.append("requires dry-run preview")
|
||||||
|
if field.two_person_approval_required:
|
||||||
|
parts.append("requires two-person approval")
|
||||||
|
if field.maintenance_required:
|
||||||
|
parts.append("requires maintenance mode")
|
||||||
|
if field.secret_handling != "none":
|
||||||
|
parts.append(f"uses {field.secret_handling} secret handling")
|
||||||
|
return "; ".join(parts) + "."
|
||||||
|
|
||||||
|
|
||||||
|
def _contains_plain_secret(value: object) -> bool:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
return False
|
||||||
|
secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"}
|
||||||
|
for key, item in value.items():
|
||||||
|
name = str(key).strip().casefold()
|
||||||
|
if name in secret_keys and item not in (None, "", {"secret_ref": ""}):
|
||||||
|
return True
|
||||||
|
if isinstance(item, Mapping) and _contains_plain_secret(item):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
@@ -2,9 +2,36 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from contextvars import ContextVar
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
import re
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
_TRACE_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$")
|
||||||
|
|
||||||
|
|
||||||
|
def new_event_id() -> str:
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_trace_id(value: str | None) -> str | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
candidate = value.strip()
|
||||||
|
return candidate if _TRACE_ID_RE.fullmatch(candidate) else None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class EventTrace:
|
||||||
|
correlation_id: str
|
||||||
|
causation_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
_current_trace: ContextVar[EventTrace | None] = ContextVar("govoplan_event_trace", default=None)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -13,11 +40,59 @@ class PlatformEvent:
|
|||||||
module_id: str
|
module_id: str
|
||||||
payload: Mapping[str, Any] = field(default_factory=dict)
|
payload: Mapping[str, Any] = field(default_factory=dict)
|
||||||
occurred_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
occurred_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||||
|
event_id: str = field(default_factory=new_event_id)
|
||||||
|
correlation_id: str | None = None
|
||||||
|
causation_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
EventHandler = Callable[[PlatformEvent], None]
|
EventHandler = Callable[[PlatformEvent], None]
|
||||||
|
|
||||||
|
|
||||||
|
def current_event_trace() -> EventTrace | None:
|
||||||
|
return _current_trace.get()
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def event_context(
|
||||||
|
*,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
causation_id: str | None = None,
|
||||||
|
):
|
||||||
|
parent = current_event_trace()
|
||||||
|
trace = EventTrace(
|
||||||
|
correlation_id=normalize_trace_id(correlation_id) or (parent.correlation_id if parent else new_event_id()),
|
||||||
|
causation_id=normalize_trace_id(causation_id) or (parent.causation_id if parent else None),
|
||||||
|
)
|
||||||
|
token = _current_trace.set(trace)
|
||||||
|
try:
|
||||||
|
yield trace
|
||||||
|
finally:
|
||||||
|
_current_trace.reset(token)
|
||||||
|
|
||||||
|
|
||||||
|
def event_trace_for(event: PlatformEvent) -> EventTrace:
|
||||||
|
active = current_event_trace()
|
||||||
|
return EventTrace(
|
||||||
|
correlation_id=normalize_trace_id(event.correlation_id) or (active.correlation_id if active else event.event_id),
|
||||||
|
causation_id=normalize_trace_id(event.causation_id) or (active.causation_id if active else None),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_event_trace(event: PlatformEvent) -> PlatformEvent:
|
||||||
|
trace = event_trace_for(event)
|
||||||
|
if event.correlation_id == trace.correlation_id and event.causation_id == trace.causation_id:
|
||||||
|
return event
|
||||||
|
return PlatformEvent(
|
||||||
|
type=event.type,
|
||||||
|
module_id=event.module_id,
|
||||||
|
payload=event.payload,
|
||||||
|
occurred_at=event.occurred_at,
|
||||||
|
event_id=event.event_id,
|
||||||
|
correlation_id=trace.correlation_id,
|
||||||
|
causation_id=trace.causation_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class EventBus:
|
class EventBus:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self._subscribers: dict[str, list[EventHandler]] = defaultdict(list)
|
self._subscribers: dict[str, list[EventHandler]] = defaultdict(list)
|
||||||
@@ -26,8 +101,9 @@ class EventBus:
|
|||||||
self._subscribers[event_type].append(handler)
|
self._subscribers[event_type].append(handler)
|
||||||
|
|
||||||
def publish(self, event: PlatformEvent) -> None:
|
def publish(self, event: PlatformEvent) -> None:
|
||||||
for handler in self._subscribers.get(event.type, ()):
|
traced = ensure_event_trace(event)
|
||||||
handler(event)
|
with event_context(correlation_id=traced.correlation_id, causation_id=traced.event_id):
|
||||||
for handler in self._subscribers.get("*", ()):
|
for handler in self._subscribers.get(traced.type, ()):
|
||||||
handler(event)
|
handler(traced)
|
||||||
|
for handler in self._subscribers.get("*", ()):
|
||||||
|
handler(traced)
|
||||||
|
|||||||
46
src/govoplan_core/core/identity.py
Normal file
46
src/govoplan_core/core/identity.py
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Literal, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
IDENTITY_MODULE_ID = "identity"
|
||||||
|
CAPABILITY_IDENTITY_DIRECTORY = "identity.directory"
|
||||||
|
|
||||||
|
IdentityStatus = Literal["active", "inactive", "suspended"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IdentityRef:
|
||||||
|
id: str
|
||||||
|
display_name: str | None = None
|
||||||
|
external_subject: str | None = None
|
||||||
|
source: str = "local"
|
||||||
|
primary_account_id: str | None = None
|
||||||
|
account_ids: tuple[str, ...] = ()
|
||||||
|
status: IdentityStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class IdentityAccountLinkRef:
|
||||||
|
id: str
|
||||||
|
identity_id: str
|
||||||
|
account_id: str
|
||||||
|
is_primary: bool = False
|
||||||
|
source: str = "local"
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class IdentityDirectory(Protocol):
|
||||||
|
def get_identity(self, identity_id: str) -> IdentityRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def identity_for_account(self, account_id: str) -> IdentityRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def identities_for_accounts(self, account_ids: Sequence[str]) -> Sequence[IdentityRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def accounts_for_identity(self, identity_id: str) -> Sequence[IdentityAccountLinkRef]:
|
||||||
|
...
|
||||||
@@ -10,6 +10,7 @@ from govoplan_core.core.module_management import ModuleManagementError, REQUIRED
|
|||||||
from govoplan_core.core.modules import ModuleContext, ModuleManifest
|
from govoplan_core.core.modules import ModuleContext, ModuleManifest
|
||||||
from govoplan_core.core.registry import PlatformRegistry
|
from govoplan_core.core.registry import PlatformRegistry
|
||||||
from govoplan_core.core.runtime import configure_runtime
|
from govoplan_core.core.runtime import configure_runtime
|
||||||
|
from govoplan_core.server.route_validation import validate_router_can_mount
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -129,6 +130,7 @@ class ModuleLifecycleManager:
|
|||||||
module_router = manifest.route_factory(self.context)
|
module_router = manifest.route_factory(self.context)
|
||||||
guarded = APIRouter(dependencies=[Depends(require_module_active(module_id))])
|
guarded = APIRouter(dependencies=[Depends(require_module_active(module_id))])
|
||||||
guarded.include_router(module_router)
|
guarded.include_router(module_router)
|
||||||
|
validate_router_can_mount(app, guarded, prefix=self.api_prefix, owner=f"module {module_id!r}")
|
||||||
app.include_router(guarded, prefix=self.api_prefix)
|
app.include_router(guarded, prefix=self.api_prefix)
|
||||||
app.openapi_schema = None
|
app.openapi_schema = None
|
||||||
self._mounted_modules.add(module_id)
|
self._mounted_modules.add(module_id)
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Iterable, Mapping
|
from collections.abc import Iterable, Mapping
|
||||||
from contextlib import AbstractContextManager
|
from contextlib import AbstractContextManager, closing
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from importlib import metadata
|
from importlib import metadata
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -22,6 +23,7 @@ import urllib.request
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.core.maintenance import saved_maintenance_mode
|
from govoplan_core.core.maintenance import saved_maintenance_mode
|
||||||
|
from govoplan_core.core.events import current_event_trace
|
||||||
from govoplan_core.core.module_management import (
|
from govoplan_core.core.module_management import (
|
||||||
PROTECTED_MODULES,
|
PROTECTED_MODULES,
|
||||||
ModuleInstallPlan,
|
ModuleInstallPlan,
|
||||||
@@ -85,6 +87,7 @@ class ModuleInstallerPreflight:
|
|||||||
rollback_commands: tuple[str, ...]
|
rollback_commands: tuple[str, ...]
|
||||||
issues: tuple[ModuleInstallerIssue, ...]
|
issues: tuple[ModuleInstallerIssue, ...]
|
||||||
checklist: tuple[ModuleInstallChecklistItem, ...] = ()
|
checklist: tuple[ModuleInstallChecklistItem, ...] = ()
|
||||||
|
artifact_integrity: tuple[dict[str, object], ...] = ()
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, object]:
|
def as_dict(self) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
@@ -96,6 +99,7 @@ class ModuleInstallerPreflight:
|
|||||||
"rollback_commands": list(self.rollback_commands),
|
"rollback_commands": list(self.rollback_commands),
|
||||||
"issues": [issue.as_dict() for issue in self.issues],
|
"issues": [issue.as_dict() for issue in self.issues],
|
||||||
"checklist": [item.as_dict() for item in self.checklist],
|
"checklist": [item.as_dict() for item in self.checklist],
|
||||||
|
"artifact_integrity": list(self.artifact_integrity),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -203,6 +207,12 @@ def module_install_preflight(
|
|||||||
if item.webui_package or item.webui_ref:
|
if item.webui_package or item.webui_ref:
|
||||||
frontend_rebuild_required = True
|
frontend_rebuild_required = True
|
||||||
|
|
||||||
|
artifact_integrity, artifact_integrity_issues = _verify_artifact_integrity(
|
||||||
|
planned_items,
|
||||||
|
require_verified=_configured_require_artifact_integrity(),
|
||||||
|
)
|
||||||
|
issues.extend(artifact_integrity_issues)
|
||||||
|
|
||||||
if frontend_rebuild_required:
|
if frontend_rebuild_required:
|
||||||
if webui_root is None:
|
if webui_root is None:
|
||||||
issues.append(ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build."))
|
issues.append(ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build."))
|
||||||
@@ -229,6 +239,7 @@ def module_install_preflight(
|
|||||||
rollback_commands=rollback_commands,
|
rollback_commands=rollback_commands,
|
||||||
issues=tuple(issues),
|
issues=tuple(issues),
|
||||||
checklist=checklist,
|
checklist=checklist,
|
||||||
|
artifact_integrity=artifact_integrity,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -251,6 +262,7 @@ def run_module_install_plan(
|
|||||||
activate_installed_modules: bool = True,
|
activate_installed_modules: bool = True,
|
||||||
remove_uninstalled_modules_from_desired: bool = True,
|
remove_uninstalled_modules_from_desired: bool = True,
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
|
request_context: Mapping[str, object] | None = None,
|
||||||
) -> ModuleInstallerRunResult:
|
) -> ModuleInstallerRunResult:
|
||||||
maintenance_mode = saved_maintenance_mode(session)
|
maintenance_mode = saved_maintenance_mode(session)
|
||||||
effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url)
|
effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url)
|
||||||
@@ -302,6 +314,8 @@ def run_module_install_plan(
|
|||||||
"activate_installed_modules": activate_installed_modules,
|
"activate_installed_modules": activate_installed_modules,
|
||||||
"remove_uninstalled_modules_from_desired": remove_uninstalled_modules_from_desired,
|
"remove_uninstalled_modules_from_desired": remove_uninstalled_modules_from_desired,
|
||||||
}
|
}
|
||||||
|
if request_context:
|
||||||
|
record["request"] = dict(request_context)
|
||||||
_write_json(record_path, record)
|
_write_json(record_path, record)
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
@@ -409,6 +423,7 @@ def supervise_module_install_plan(
|
|||||||
health_urls: Iterable[str] | None = None,
|
health_urls: Iterable[str] | None = None,
|
||||||
health_timeout_seconds: float = 60.0,
|
health_timeout_seconds: float = 60.0,
|
||||||
health_interval_seconds: float = 2.0,
|
health_interval_seconds: float = 2.0,
|
||||||
|
request_context: Mapping[str, object] | None = None,
|
||||||
) -> ModuleInstallerRunResult:
|
) -> ModuleInstallerRunResult:
|
||||||
effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url)
|
effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url)
|
||||||
effective_restart_commands = _command_list(restart_command, restart_commands)
|
effective_restart_commands = _command_list(restart_command, restart_commands)
|
||||||
@@ -431,6 +446,7 @@ def supervise_module_install_plan(
|
|||||||
activate_installed_modules=activate_installed_modules,
|
activate_installed_modules=activate_installed_modules,
|
||||||
remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired,
|
remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired,
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
|
request_context=request_context,
|
||||||
)
|
)
|
||||||
supervisor: dict[str, object] = {
|
supervisor: dict[str, object] = {
|
||||||
"started_at": datetime.now(tz=UTC).isoformat(),
|
"started_at": datetime.now(tz=UTC).isoformat(),
|
||||||
@@ -647,8 +663,10 @@ def queue_module_installer_request(
|
|||||||
options: Mapping[str, object] | None = None,
|
options: Mapping[str, object] | None = None,
|
||||||
requested_by: str | None = None,
|
requested_by: str | None = None,
|
||||||
retry_of: str | None = None,
|
retry_of: str | None = None,
|
||||||
|
trace: Mapping[str, object] | None = None,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
request_id = _run_id()
|
request_id = _run_id()
|
||||||
|
effective_trace = dict(trace) if trace is not None else _current_trace_payload()
|
||||||
record = {
|
record = {
|
||||||
"request_id": request_id,
|
"request_id": request_id,
|
||||||
"status": "queued",
|
"status": "queued",
|
||||||
@@ -656,6 +674,8 @@ def queue_module_installer_request(
|
|||||||
"requested_by": requested_by,
|
"requested_by": requested_by,
|
||||||
"options": dict(options or {}),
|
"options": dict(options or {}),
|
||||||
}
|
}
|
||||||
|
if effective_trace:
|
||||||
|
record["trace"] = effective_trace
|
||||||
if retry_of:
|
if retry_of:
|
||||||
record["retry_of"] = retry_of
|
record["retry_of"] = retry_of
|
||||||
with _request_queue_lock(runtime_dir):
|
with _request_queue_lock(runtime_dir):
|
||||||
@@ -1273,6 +1293,7 @@ def _run_summary(path: Path, record: Mapping[str, object]) -> dict[str, object]:
|
|||||||
supervisor = record.get("supervisor") if isinstance(record.get("supervisor"), Mapping) else {}
|
supervisor = record.get("supervisor") if isinstance(record.get("supervisor"), Mapping) else {}
|
||||||
commands = record.get("commands") if isinstance(record.get("commands"), list) else []
|
commands = record.get("commands") if isinstance(record.get("commands"), list) else []
|
||||||
plan = record.get("plan") if isinstance(record.get("plan"), list) else []
|
plan = record.get("plan") if isinstance(record.get("plan"), list) else []
|
||||||
|
request = record.get("request") if isinstance(record.get("request"), Mapping) else {}
|
||||||
return {
|
return {
|
||||||
"run_id": str(record.get("run_id") or path.parent.name),
|
"run_id": str(record.get("run_id") or path.parent.name),
|
||||||
"status": str(record.get("status") or "unknown"),
|
"status": str(record.get("status") or "unknown"),
|
||||||
@@ -1284,9 +1305,22 @@ def _run_summary(path: Path, record: Mapping[str, object]) -> dict[str, object]:
|
|||||||
"record_path": str(path),
|
"record_path": str(path),
|
||||||
"commands_count": len(commands),
|
"commands_count": len(commands),
|
||||||
"planned_modules": [str(item.get("module_id")) for item in plan if isinstance(item, Mapping) and item.get("module_id")],
|
"planned_modules": [str(item.get("module_id")) for item in plan if isinstance(item, Mapping) and item.get("module_id")],
|
||||||
|
"request_id": request.get("request_id") if isinstance(request, Mapping) else None,
|
||||||
|
"requested_by": request.get("requested_by") if isinstance(request, Mapping) else None,
|
||||||
|
"trace": request.get("trace") if isinstance(request.get("trace"), Mapping) else None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _current_trace_payload() -> dict[str, object]:
|
||||||
|
trace = current_event_trace()
|
||||||
|
if trace is None:
|
||||||
|
return {}
|
||||||
|
payload: dict[str, object] = {"correlation_id": trace.correlation_id}
|
||||||
|
if trace.causation_id:
|
||||||
|
payload["causation_id"] = trace.causation_id
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
def _database_restore_was_applied(record: Mapping[str, object]) -> bool:
|
def _database_restore_was_applied(record: Mapping[str, object]) -> bool:
|
||||||
restore = record.get("rollback_database_restore")
|
restore = record.get("rollback_database_restore")
|
||||||
return isinstance(restore, Mapping) and restore.get("restored") is True
|
return isinstance(restore, Mapping) and restore.get("restored") is True
|
||||||
@@ -1305,6 +1339,156 @@ def _looks_pinned_dependency_ref(value: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_require_artifact_integrity() -> bool:
|
||||||
|
return os.getenv("GOVOPLAN_MODULE_INSTALLER_REQUIRE_ARTIFACT_INTEGRITY", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_artifact_integrity(
|
||||||
|
planned_items: tuple[ModuleInstallPlanItem, ...],
|
||||||
|
*,
|
||||||
|
require_verified: bool,
|
||||||
|
) -> tuple[tuple[dict[str, object], ...], tuple[ModuleInstallerIssue, ...]]:
|
||||||
|
records: list[dict[str, object]] = []
|
||||||
|
issues: list[ModuleInstallerIssue] = []
|
||||||
|
for item in planned_items:
|
||||||
|
if item.action != "install":
|
||||||
|
continue
|
||||||
|
for kind, package_name, package_ref in (
|
||||||
|
("python", item.python_package, item.python_ref),
|
||||||
|
("webui", item.webui_package, item.webui_ref),
|
||||||
|
):
|
||||||
|
if not package_ref:
|
||||||
|
continue
|
||||||
|
raw_metadata = _artifact_metadata(item.artifact_integrity, kind)
|
||||||
|
if raw_metadata is None:
|
||||||
|
if require_verified:
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker",
|
||||||
|
"artifact_integrity_required",
|
||||||
|
f"{kind.capitalize()} artifact integrity metadata is required in production mode.",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
continue
|
||||||
|
record, record_issues = _verify_artifact_metadata(
|
||||||
|
item,
|
||||||
|
kind=kind,
|
||||||
|
package_name=package_name,
|
||||||
|
package_ref=package_ref,
|
||||||
|
metadata=raw_metadata,
|
||||||
|
require_verified=require_verified,
|
||||||
|
)
|
||||||
|
records.append(record)
|
||||||
|
issues.extend(record_issues)
|
||||||
|
return tuple(records), tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_metadata(value: Mapping[str, object] | None, kind: str) -> Mapping[str, object] | None:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
return None
|
||||||
|
raw = value.get(kind)
|
||||||
|
return raw if isinstance(raw, Mapping) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_artifact_metadata(
|
||||||
|
item: ModuleInstallPlanItem,
|
||||||
|
*,
|
||||||
|
kind: str,
|
||||||
|
package_name: str | None,
|
||||||
|
package_ref: str,
|
||||||
|
metadata: Mapping[str, object],
|
||||||
|
require_verified: bool,
|
||||||
|
) -> tuple[dict[str, object], tuple[ModuleInstallerIssue, ...]]:
|
||||||
|
issues: list[ModuleInstallerIssue] = []
|
||||||
|
expected_sha256 = _artifact_text(metadata, "sha256")
|
||||||
|
artifact_path = _artifact_path(metadata)
|
||||||
|
expected_ref = _artifact_text(metadata, "ref") or _artifact_text(metadata, "expected_ref")
|
||||||
|
record: dict[str, object] = {
|
||||||
|
"module_id": item.module_id,
|
||||||
|
"kind": kind,
|
||||||
|
"package_ref": package_ref,
|
||||||
|
"verified": False,
|
||||||
|
}
|
||||||
|
if package_name:
|
||||||
|
record["package"] = package_name
|
||||||
|
for key in ("sha256", "sbom_url", "provenance_url", "registry_identity", "git_ref"):
|
||||||
|
value = _artifact_text(metadata, key)
|
||||||
|
if value:
|
||||||
|
record[key] = value
|
||||||
|
if expected_ref:
|
||||||
|
record["expected_ref"] = expected_ref
|
||||||
|
if expected_ref != package_ref:
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker",
|
||||||
|
"artifact_ref_mismatch",
|
||||||
|
f"{kind.capitalize()} artifact metadata expected ref {expected_ref!r} but the install plan uses {package_ref!r}.",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
if not expected_sha256:
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker" if require_verified else "warning",
|
||||||
|
"artifact_digest_missing",
|
||||||
|
f"{kind.capitalize()} artifact integrity metadata has no sha256 digest.",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
return record, tuple(issues)
|
||||||
|
if artifact_path is None:
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker" if require_verified else "warning",
|
||||||
|
"artifact_not_verified",
|
||||||
|
f"{kind.capitalize()} artifact digest is declared but no local artifact path is available for verification.",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
return record, tuple(issues)
|
||||||
|
record["artifact_path"] = str(artifact_path)
|
||||||
|
if not artifact_path.exists():
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker",
|
||||||
|
"artifact_missing",
|
||||||
|
f"{kind.capitalize()} artifact does not exist: {artifact_path}",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
return record, tuple(issues)
|
||||||
|
actual_sha256 = _sha256_file(artifact_path)
|
||||||
|
record["actual_sha256"] = actual_sha256
|
||||||
|
if actual_sha256 != expected_sha256.lower():
|
||||||
|
issues.append(ModuleInstallerIssue(
|
||||||
|
"blocker",
|
||||||
|
"artifact_digest_mismatch",
|
||||||
|
f"{kind.capitalize()} artifact digest mismatch for {artifact_path}.",
|
||||||
|
item.module_id,
|
||||||
|
))
|
||||||
|
return record, tuple(issues)
|
||||||
|
record["verified"] = True
|
||||||
|
record["verified_at"] = datetime.now(tz=UTC).isoformat()
|
||||||
|
return record, tuple(issues)
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_text(metadata: Mapping[str, object], key: str) -> str | None:
|
||||||
|
value = metadata.get(key)
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
text = str(value).strip()
|
||||||
|
return text or None
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact_path(metadata: Mapping[str, object]) -> Path | None:
|
||||||
|
value = _artifact_text(metadata, "path") or _artifact_text(metadata, "artifact_path")
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
if value.startswith("file://"):
|
||||||
|
value = value[len("file://"):]
|
||||||
|
path = Path(value).expanduser()
|
||||||
|
return path if path.is_absolute() else Path.cwd() / path
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _snapshot_environment(
|
def _snapshot_environment(
|
||||||
run_dir: Path,
|
run_dir: Path,
|
||||||
*,
|
*,
|
||||||
@@ -1368,7 +1552,7 @@ def _snapshot_sqlite_database(run_dir: Path, database_url: str | None) -> dict[s
|
|||||||
raise ModuleInstallerError("Automatic database backup for --migrate currently supports sqlite:/// URLs only.")
|
raise ModuleInstallerError("Automatic database backup for --migrate currently supports sqlite:/// URLs only.")
|
||||||
backup_path = run_dir / "database.sqlite.before"
|
backup_path = run_dir / "database.sqlite.before"
|
||||||
if db_path.exists():
|
if db_path.exists():
|
||||||
with sqlite3.connect(str(db_path)) as source, sqlite3.connect(str(backup_path)) as target:
|
with closing(sqlite3.connect(str(db_path))) as source, closing(sqlite3.connect(str(backup_path))) as target:
|
||||||
source.backup(target)
|
source.backup(target)
|
||||||
else:
|
else:
|
||||||
backup_path.touch()
|
backup_path.touch()
|
||||||
@@ -1467,7 +1651,7 @@ def _restore_database_snapshot(
|
|||||||
if backup_path.stat().st_size == 0:
|
if backup_path.stat().st_size == 0:
|
||||||
target_path.unlink(missing_ok=True)
|
target_path.unlink(missing_ok=True)
|
||||||
return {"restored": True, "type": "sqlite", "source": str(target_path), "empty": True}
|
return {"restored": True, "type": "sqlite", "source": str(target_path), "empty": True}
|
||||||
with sqlite3.connect(str(backup_path)) as source, sqlite3.connect(str(target_path)) as target:
|
with closing(sqlite3.connect(str(backup_path))) as source, closing(sqlite3.connect(str(target_path))) as target:
|
||||||
source.backup(target)
|
source.backup(target)
|
||||||
return {"restored": True, "type": "sqlite", "source": str(target_path)}
|
return {"restored": True, "type": "sqlite", "source": str(target_path)}
|
||||||
|
|
||||||
@@ -1523,6 +1707,9 @@ def _run_database_hook(
|
|||||||
})
|
})
|
||||||
if database_url:
|
if database_url:
|
||||||
env["GOVOPLAN_DATABASE_URL"] = database_url
|
env["GOVOPLAN_DATABASE_URL"] = database_url
|
||||||
|
pgtools_url = _database_url_for_pgtools(database_url)
|
||||||
|
if pgtools_url:
|
||||||
|
env["GOVOPLAN_DATABASE_URL_PGTOOLS"] = pgtools_url
|
||||||
env.setdefault("DATABASE_URL", database_url)
|
env.setdefault("DATABASE_URL", database_url)
|
||||||
return subprocess.run(
|
return subprocess.run(
|
||||||
command,
|
command,
|
||||||
@@ -1535,6 +1722,14 @@ def _run_database_hook(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _database_url_for_pgtools(database_url: str) -> str | None:
|
||||||
|
if database_url.startswith(("postgresql://", "postgres://")):
|
||||||
|
return database_url
|
||||||
|
if database_url.startswith("postgresql+"):
|
||||||
|
return re.sub(r"^postgresql\+[^:]+://", "postgresql://", database_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _sqlite_database_path(database_url: str | None) -> Path | None:
|
def _sqlite_database_path(database_url: str | None) -> Path | None:
|
||||||
if not database_url or not database_url.startswith("sqlite:///"):
|
if not database_url or not database_url.startswith("sqlite:///"):
|
||||||
return None
|
return None
|
||||||
@@ -1572,6 +1767,7 @@ def _mark_applied(item: ModuleInstallPlanItem) -> ModuleInstallPlanItem:
|
|||||||
python_ref=item.python_ref,
|
python_ref=item.python_ref,
|
||||||
webui_package=item.webui_package,
|
webui_package=item.webui_package,
|
||||||
webui_ref=item.webui_ref,
|
webui_ref=item.webui_ref,
|
||||||
|
artifact_integrity=item.artifact_integrity,
|
||||||
destroy_data=item.destroy_data,
|
destroy_data=item.destroy_data,
|
||||||
status="applied",
|
status="applied",
|
||||||
notes=item.notes,
|
notes=item.notes,
|
||||||
|
|||||||
@@ -11,14 +11,103 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
from cryptography.exceptions import InvalidSignature
|
from cryptography.exceptions import InvalidSignature
|
||||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
from cryptography.hazmat.primitives import serialization
|
||||||
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
||||||
|
|
||||||
|
|
||||||
def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]:
|
def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]:
|
||||||
|
return _module_license_decision_from_validation(validate_module_license(), required_features)
|
||||||
|
|
||||||
|
|
||||||
|
def module_license_diagnostics(
|
||||||
|
path: Path | None = None,
|
||||||
|
*,
|
||||||
|
required_features: list[str] | tuple[str, ...] = (),
|
||||||
|
trusted_keys: dict[str, str] | None = None,
|
||||||
|
require_trusted: bool | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
validation = validate_module_license(path, trusted_keys=trusted_keys, require_trusted=require_trusted)
|
||||||
|
decision = _module_license_decision_from_validation(validation, required_features)
|
||||||
|
guidance = _license_guidance(validation, decision)
|
||||||
|
return {
|
||||||
|
"configured": validation["configured"],
|
||||||
|
"valid": validation["valid"],
|
||||||
|
"path": validation["path"],
|
||||||
|
"license_id": validation["license_id"],
|
||||||
|
"subject": validation["subject"],
|
||||||
|
"features": validation["features"],
|
||||||
|
"valid_from": validation["valid_from"],
|
||||||
|
"valid_until": validation["valid_until"],
|
||||||
|
"signed": validation["signed"],
|
||||||
|
"trusted": validation["trusted"],
|
||||||
|
"key_id": validation["key_id"],
|
||||||
|
"enforced": _license_enforcement_enabled(),
|
||||||
|
"allowed": decision["allowed"],
|
||||||
|
"required_features": decision["required_features"],
|
||||||
|
"missing_features": decision["missing_features"],
|
||||||
|
"expires_in_days": _expires_in_days(validation.get("valid_until")),
|
||||||
|
"reason": decision.get("reason") or validation.get("error"),
|
||||||
|
"guidance": guidance,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def issue_module_license(
|
||||||
|
*,
|
||||||
|
path: Path,
|
||||||
|
license_id: str,
|
||||||
|
subject: str,
|
||||||
|
features: list[str] | tuple[str, ...],
|
||||||
|
valid_until: str | datetime,
|
||||||
|
signing_key_id: str,
|
||||||
|
signing_private_key_path: Path,
|
||||||
|
valid_from: str | datetime | None = None,
|
||||||
|
issuer: str | None = None,
|
||||||
|
notes: str | None = None,
|
||||||
|
) -> Path:
|
||||||
|
clean_license_id = str(license_id).strip()
|
||||||
|
clean_subject = str(subject).strip()
|
||||||
|
clean_key_id = str(signing_key_id).strip()
|
||||||
|
clean_features = _normalize_required_features(features)
|
||||||
|
if not clean_license_id:
|
||||||
|
raise ValueError("License id is required.")
|
||||||
|
if not clean_subject:
|
||||||
|
raise ValueError("License subject is required.")
|
||||||
|
if not clean_key_id:
|
||||||
|
raise ValueError("License signing key id is required.")
|
||||||
|
if not clean_features:
|
||||||
|
raise ValueError("At least one license feature is required.")
|
||||||
|
payload: dict[str, object] = {
|
||||||
|
"license_id": clean_license_id,
|
||||||
|
"subject": clean_subject,
|
||||||
|
"features": list(clean_features),
|
||||||
|
"valid_from": _datetime_text(valid_from or datetime.now(tz=UTC)),
|
||||||
|
"valid_until": _datetime_text(valid_until),
|
||||||
|
}
|
||||||
|
clean_issuer = str(issuer).strip() if issuer else ""
|
||||||
|
if clean_issuer:
|
||||||
|
payload["issuer"] = clean_issuer
|
||||||
|
clean_notes = str(notes).strip() if notes else ""
|
||||||
|
if clean_notes:
|
||||||
|
payload["notes"] = clean_notes
|
||||||
|
private_key = _load_private_key(signing_private_key_path)
|
||||||
|
signature = private_key.sign(_canonical_bytes(payload))
|
||||||
|
payload["signature"] = {
|
||||||
|
"algorithm": "ed25519",
|
||||||
|
"key_id": clean_key_id,
|
||||||
|
"value": base64.b64encode(signature).decode("ascii"),
|
||||||
|
}
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _module_license_decision_from_validation(
|
||||||
|
validation: dict[str, object],
|
||||||
|
required_features: list[str] | tuple[str, ...],
|
||||||
|
) -> dict[str, object]:
|
||||||
required = tuple(dict.fromkeys(str(item).strip() for item in required_features if str(item).strip()))
|
required = tuple(dict.fromkeys(str(item).strip() for item in required_features if str(item).strip()))
|
||||||
if not required:
|
if not required:
|
||||||
return {"allowed": True, "required_features": [], "missing_features": [], "enforced": False}
|
return {"allowed": True, "required_features": [], "missing_features": [], "enforced": False}
|
||||||
validation = validate_module_license()
|
|
||||||
enforced = _license_enforcement_enabled()
|
enforced = _license_enforcement_enabled()
|
||||||
if not validation["valid"]:
|
if not validation["valid"]:
|
||||||
return {
|
return {
|
||||||
@@ -102,6 +191,43 @@ def _license_result(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _license_guidance(validation: dict[str, object], decision: dict[str, object]) -> list[str]:
|
||||||
|
guidance: list[str] = []
|
||||||
|
if not validation.get("configured"):
|
||||||
|
guidance.append("Configure GOVOPLAN_LICENSE_FILE with the current signed license file.")
|
||||||
|
elif validation.get("path") and not validation.get("valid") and validation.get("error"):
|
||||||
|
guidance.append("Import a renewed license file or correct the configured license path before enforcing installs.")
|
||||||
|
if validation.get("configured") and not validation.get("signed"):
|
||||||
|
guidance.append("Use a signed license for production and configure GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE.")
|
||||||
|
elif validation.get("signed") and not validation.get("trusted"):
|
||||||
|
guidance.append("Add the signing public key to the trusted license keyring, or rotate to a trusted issuer key.")
|
||||||
|
expires_in_days = _expires_in_days(validation.get("valid_until"))
|
||||||
|
if expires_in_days is not None:
|
||||||
|
if expires_in_days < 0:
|
||||||
|
guidance.append("Renew the license; the validity window has ended.")
|
||||||
|
elif expires_in_days <= 30:
|
||||||
|
guidance.append(f"Renew the license soon; it expires in {expires_in_days} day(s).")
|
||||||
|
missing = decision.get("missing_features")
|
||||||
|
if isinstance(missing, list) and missing:
|
||||||
|
guidance.append("Request a renewal or entitlement update for: " + ", ".join(str(item) for item in missing))
|
||||||
|
if not _license_enforcement_enabled():
|
||||||
|
guidance.append("License enforcement is observe-only until GOVOPLAN_LICENSE_ENFORCEMENT=true is set.")
|
||||||
|
return guidance
|
||||||
|
|
||||||
|
|
||||||
|
def _expires_in_days(value: object) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
expires_at = _parse_datetime(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if expires_at is None:
|
||||||
|
return None
|
||||||
|
seconds = (expires_at - datetime.now(tz=UTC)).total_seconds()
|
||||||
|
return int(seconds // 86400)
|
||||||
|
|
||||||
|
|
||||||
def _configured_license_path() -> Path | None:
|
def _configured_license_path() -> Path | None:
|
||||||
value = os.getenv("GOVOPLAN_LICENSE_FILE", "").strip()
|
value = os.getenv("GOVOPLAN_LICENSE_FILE", "").strip()
|
||||||
return Path(value).expanduser() if value else None
|
return Path(value).expanduser() if value else None
|
||||||
@@ -203,6 +329,13 @@ def _canonical_bytes(payload: object) -> bytes:
|
|||||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _datetime_text(value: str | datetime) -> str:
|
||||||
|
parsed = _parse_datetime(value)
|
||||||
|
if parsed is None:
|
||||||
|
raise ValueError("License validity timestamp is required.")
|
||||||
|
return parsed.isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
def _parse_datetime(value: object) -> datetime | None:
|
def _parse_datetime(value: object) -> datetime | None:
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
@@ -223,3 +356,14 @@ def _string_list(value: object) -> list[str]:
|
|||||||
if not isinstance(value, list):
|
if not isinstance(value, list):
|
||||||
raise ValueError("License features must be a list.")
|
raise ValueError("License features must be a list.")
|
||||||
return [str(item).strip() for item in value if str(item).strip()]
|
return [str(item).strip() for item in value if str(item).strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_required_features(value: list[str] | tuple[str, ...]) -> tuple[str, ...]:
|
||||||
|
return tuple(dict.fromkeys(str(item).strip() for item in value if str(item).strip()))
|
||||||
|
|
||||||
|
|
||||||
|
def _load_private_key(path: Path) -> Ed25519PrivateKey:
|
||||||
|
private_key = serialization.load_pem_private_key(path.expanduser().read_bytes(), password=None)
|
||||||
|
if not isinstance(private_key, Ed25519PrivateKey):
|
||||||
|
raise ValueError("License signing key must be an Ed25519 private key.")
|
||||||
|
return private_key
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ class ModuleInstallPlanItem:
|
|||||||
python_ref: str | None = None
|
python_ref: str | None = None
|
||||||
webui_package: str | None = None
|
webui_package: str | None = None
|
||||||
webui_ref: str | None = None
|
webui_ref: str | None = None
|
||||||
|
artifact_integrity: Mapping[str, object] | None = None
|
||||||
destroy_data: bool = False
|
destroy_data: bool = False
|
||||||
status: str = "planned"
|
status: str = "planned"
|
||||||
notes: str | None = None
|
notes: str | None = None
|
||||||
@@ -58,6 +59,8 @@ class ModuleInstallPlanItem:
|
|||||||
value = getattr(self, key)
|
value = getattr(self, key)
|
||||||
if value:
|
if value:
|
||||||
payload[key] = value
|
payload[key] = value
|
||||||
|
if self.artifact_integrity:
|
||||||
|
payload["artifact_integrity"] = dict(self.artifact_integrity)
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
@@ -113,7 +116,7 @@ def load_startup_enabled_modules(
|
|||||||
with get_database().session() as session:
|
with get_database().session() as session:
|
||||||
desired = saved_desired_enabled_modules(session, fallback)
|
desired = saved_desired_enabled_modules(session, fallback)
|
||||||
except (RuntimeError, SQLAlchemyError):
|
except (RuntimeError, SQLAlchemyError):
|
||||||
return fallback
|
desired = fallback
|
||||||
if available is None:
|
if available is None:
|
||||||
return desired
|
return desired
|
||||||
|
|
||||||
@@ -293,6 +296,7 @@ def normalize_module_install_plan_item(
|
|||||||
python_ref = _clean_optional_string(raw.get("python_ref"))
|
python_ref = _clean_optional_string(raw.get("python_ref"))
|
||||||
webui_package = _clean_optional_string(raw.get("webui_package"))
|
webui_package = _clean_optional_string(raw.get("webui_package"))
|
||||||
webui_ref = _clean_optional_string(raw.get("webui_ref"))
|
webui_ref = _clean_optional_string(raw.get("webui_ref"))
|
||||||
|
artifact_integrity = _clean_optional_mapping(raw.get("artifact_integrity"), field="artifact_integrity", module_id=module_id)
|
||||||
destroy_data = _clean_bool(raw.get("destroy_data"))
|
destroy_data = _clean_bool(raw.get("destroy_data"))
|
||||||
notes = _clean_optional_string(raw.get("notes"))
|
notes = _clean_optional_string(raw.get("notes"))
|
||||||
|
|
||||||
@@ -322,6 +326,7 @@ def normalize_module_install_plan_item(
|
|||||||
python_ref=python_ref,
|
python_ref=python_ref,
|
||||||
webui_package=webui_package,
|
webui_package=webui_package,
|
||||||
webui_ref=webui_ref,
|
webui_ref=webui_ref,
|
||||||
|
artifact_integrity=artifact_integrity,
|
||||||
destroy_data=destroy_data,
|
destroy_data=destroy_data,
|
||||||
status=status,
|
status=status,
|
||||||
notes=notes,
|
notes=notes,
|
||||||
@@ -413,6 +418,14 @@ def _clean_bool(value: object) -> bool:
|
|||||||
return bool(value)
|
return bool(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_optional_mapping(value: object, *, field: str, module_id: str) -> dict[str, object] | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ModuleManagementError(f"Install plan item {module_id!r} field {field} must be an object.")
|
||||||
|
return {str(key): item for key, item in value.items() if str(key).strip()}
|
||||||
|
|
||||||
|
|
||||||
def _validate_dependency_ref(value: str, *, field: str, module_id: str) -> None:
|
def _validate_dependency_ref(value: str, *, field: str, module_id: str) -> None:
|
||||||
lowered = value.lower()
|
lowered = value.lower()
|
||||||
if lowered.startswith(LOCAL_DEPENDENCY_REF_PREFIXES) or value.startswith(("/", "./", "../", "~")):
|
if lowered.startswith(LOCAL_DEPENDENCY_REF_PREFIXES) or value.startswith(("/", "./", "../", "~")):
|
||||||
|
|||||||
@@ -54,10 +54,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source),
|
"path": str(catalog_source),
|
||||||
"source": str(catalog_source),
|
"source": str(catalog_source),
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": False,
|
||||||
|
"cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None,
|
||||||
"modules": [],
|
"modules": [],
|
||||||
"channel": None,
|
"channel": None,
|
||||||
"sequence": None,
|
"sequence": None,
|
||||||
"generated_at": None,
|
"generated_at": None,
|
||||||
|
"not_before": None,
|
||||||
"expires_at": None,
|
"expires_at": None,
|
||||||
"signed": False,
|
"signed": False,
|
||||||
"trusted": False,
|
"trusted": False,
|
||||||
@@ -66,12 +69,14 @@ def validate_module_package_catalog(
|
|||||||
"error": f"Module package catalog does not exist: {catalog_source}",
|
"error": f"Module package catalog does not exist: {catalog_source}",
|
||||||
}
|
}
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
|
read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None}
|
||||||
try:
|
try:
|
||||||
payload = _read_catalog_payload(catalog_source)
|
payload, read_state = _read_catalog_payload_with_metadata(catalog_source)
|
||||||
modules = _normalize_catalog_modules(payload)
|
modules = _normalize_catalog_modules(payload)
|
||||||
channel = _catalog_channel(payload)
|
channel = _catalog_channel(payload)
|
||||||
sequence = _catalog_sequence(payload)
|
sequence = _catalog_sequence(payload)
|
||||||
generated_at = _catalog_optional_text(payload, "generated_at")
|
generated_at = _catalog_optional_text(payload, "generated_at")
|
||||||
|
not_before = _catalog_optional_text(payload, "not_before")
|
||||||
expires_at = _catalog_optional_text(payload, "expires_at")
|
expires_at = _catalog_optional_text(payload, "expires_at")
|
||||||
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys)
|
||||||
freshness = _catalog_freshness_state(payload)
|
freshness = _catalog_freshness_state(payload)
|
||||||
@@ -83,10 +88,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": [],
|
"modules": [],
|
||||||
"channel": None,
|
"channel": None,
|
||||||
"sequence": None,
|
"sequence": None,
|
||||||
"generated_at": None,
|
"generated_at": None,
|
||||||
|
"not_before": None,
|
||||||
"expires_at": None,
|
"expires_at": None,
|
||||||
"signed": False,
|
"signed": False,
|
||||||
"trusted": False,
|
"trusted": False,
|
||||||
@@ -101,10 +109,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": [],
|
"modules": [],
|
||||||
"channel": channel,
|
"channel": channel,
|
||||||
"sequence": sequence,
|
"sequence": sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": signature_state["signed"],
|
||||||
"trusted": signature_state["trusted"],
|
"trusted": signature_state["trusted"],
|
||||||
@@ -119,10 +130,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": [],
|
"modules": [],
|
||||||
"channel": channel,
|
"channel": channel,
|
||||||
"sequence": sequence,
|
"sequence": sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": signature_state["signed"],
|
||||||
"trusted": signature_state["trusted"],
|
"trusted": signature_state["trusted"],
|
||||||
@@ -137,10 +151,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": [],
|
"modules": [],
|
||||||
"channel": channel,
|
"channel": channel,
|
||||||
"sequence": sequence,
|
"sequence": sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": signature_state["signed"],
|
||||||
"trusted": False,
|
"trusted": False,
|
||||||
@@ -155,8 +172,10 @@ def validate_module_package_catalog(
|
|||||||
channel=channel,
|
channel=channel,
|
||||||
sequence=sequence,
|
sequence=sequence,
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
not_before=not_before,
|
||||||
expires_at=expires_at,
|
expires_at=expires_at,
|
||||||
signature_state=signature_state,
|
signature_state=signature_state,
|
||||||
|
read_state=read_state,
|
||||||
error=str(freshness["error"]),
|
error=str(freshness["error"]),
|
||||||
)
|
)
|
||||||
if not replay["valid"]:
|
if not replay["valid"]:
|
||||||
@@ -166,8 +185,10 @@ def validate_module_package_catalog(
|
|||||||
channel=channel,
|
channel=channel,
|
||||||
sequence=sequence,
|
sequence=sequence,
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
not_before=not_before,
|
||||||
expires_at=expires_at,
|
expires_at=expires_at,
|
||||||
signature_state=signature_state,
|
signature_state=signature_state,
|
||||||
|
read_state=read_state,
|
||||||
error=str(replay["error"]),
|
error=str(replay["error"]),
|
||||||
)
|
)
|
||||||
warnings.extend(str(item) for item in freshness.get("warnings", ()) if item)
|
warnings.extend(str(item) for item in freshness.get("warnings", ()) if item)
|
||||||
@@ -182,10 +203,13 @@ def validate_module_package_catalog(
|
|||||||
"path": str(catalog_source) if catalog_source is not None else None,
|
"path": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source": str(catalog_source) if catalog_source is not None else None,
|
"source": str(catalog_source) if catalog_source is not None else None,
|
||||||
"source_type": _catalog_source_type(catalog_source),
|
"source_type": _catalog_source_type(catalog_source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": list(modules),
|
"modules": list(modules),
|
||||||
"channel": channel,
|
"channel": channel,
|
||||||
"sequence": sequence,
|
"sequence": sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": signature_state["signed"],
|
||||||
"trusted": signature_state["trusted"],
|
"trusted": signature_state["trusted"],
|
||||||
@@ -353,14 +377,31 @@ def _parse_trusted_keys(value: str) -> dict[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def _read_catalog_payload(source: Path | str | None) -> object:
|
def _read_catalog_payload(source: Path | str | None) -> object:
|
||||||
|
payload, _metadata = _read_catalog_payload_with_metadata(source)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[object, dict[str, object]]:
|
||||||
|
cache_path = _configured_catalog_cache_path()
|
||||||
|
metadata: dict[str, object] = {
|
||||||
|
"cache_used": False,
|
||||||
|
"cache_path": str(cache_path) if cache_path is not None else None,
|
||||||
|
}
|
||||||
if source is None:
|
if source is None:
|
||||||
return {"modules": []}
|
return {"modules": []}, metadata
|
||||||
if isinstance(source, str) and _is_http_url(source):
|
if isinstance(source, str) and _is_http_url(source):
|
||||||
return json.loads(_read_catalog_url(source))
|
body, cache_used = _read_catalog_url_with_metadata(source)
|
||||||
return json.loads(Path(source).read_text(encoding="utf-8"))
|
metadata["cache_used"] = cache_used
|
||||||
|
return json.loads(body), metadata
|
||||||
|
return json.loads(Path(source).read_text(encoding="utf-8")), metadata
|
||||||
|
|
||||||
|
|
||||||
def _read_catalog_url(url: str) -> str:
|
def _read_catalog_url(url: str) -> str:
|
||||||
|
body, _cache_used = _read_catalog_url_with_metadata(url)
|
||||||
|
return body
|
||||||
|
|
||||||
|
|
||||||
|
def _read_catalog_url_with_metadata(url: str) -> tuple[str, bool]:
|
||||||
cache_path = _configured_catalog_cache_path()
|
cache_path = _configured_catalog_cache_path()
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=15) as response:
|
with urllib.request.urlopen(url, timeout=15) as response:
|
||||||
@@ -368,10 +409,10 @@ def _read_catalog_url(url: str) -> str:
|
|||||||
if cache_path is not None:
|
if cache_path is not None:
|
||||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(body, encoding="utf-8")
|
cache_path.write_text(body, encoding="utf-8")
|
||||||
return body
|
return body, False
|
||||||
except (OSError, urllib.error.URLError):
|
except (OSError, urllib.error.URLError):
|
||||||
if cache_path is not None and cache_path.exists():
|
if cache_path is not None and cache_path.exists():
|
||||||
return cache_path.read_text(encoding="utf-8")
|
return cache_path.read_text(encoding="utf-8"), True
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
@@ -549,7 +590,7 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
|||||||
action = str(value.get("action") or "install")
|
action = str(value.get("action") or "install")
|
||||||
if action not in {"install", "uninstall"}:
|
if action not in {"install", "uninstall"}:
|
||||||
raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}")
|
raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}")
|
||||||
return {
|
item = {
|
||||||
"module_id": module_id,
|
"module_id": module_id,
|
||||||
"name": str(value.get("name") or module_id),
|
"name": str(value.get("name") or module_id),
|
||||||
"description": _optional_str(value, "description"),
|
"description": _optional_str(value, "description"),
|
||||||
@@ -563,6 +604,10 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]:
|
|||||||
"notes": _optional_str(value, "notes"),
|
"notes": _optional_str(value, "notes"),
|
||||||
"tags": _string_list(value.get("tags")),
|
"tags": _string_list(value.get("tags")),
|
||||||
}
|
}
|
||||||
|
artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity"))
|
||||||
|
if artifact_integrity:
|
||||||
|
item["artifact_integrity"] = artifact_integrity
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
def _required_str(value: dict[str, Any], key: str) -> str:
|
def _required_str(value: dict[str, Any], key: str) -> str:
|
||||||
@@ -588,6 +633,37 @@ def _string_list(value: Any) -> list[str]:
|
|||||||
return [str(item).strip() for item in value if str(item).strip()]
|
return [str(item).strip() for item in value if str(item).strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_artifact_integrity(value: Any) -> dict[str, object]:
|
||||||
|
if value is None:
|
||||||
|
return {}
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise ValueError("Module package catalog artifact_integrity must be an object.")
|
||||||
|
normalized: dict[str, object] = {}
|
||||||
|
for key in ("python", "webui"):
|
||||||
|
raw = value.get(key)
|
||||||
|
if raw is None:
|
||||||
|
continue
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise ValueError(f"Module package catalog artifact_integrity.{key} must be an object.")
|
||||||
|
clean = {
|
||||||
|
field: text
|
||||||
|
for field in (
|
||||||
|
"ref",
|
||||||
|
"path",
|
||||||
|
"artifact_path",
|
||||||
|
"sha256",
|
||||||
|
"sbom_url",
|
||||||
|
"provenance_url",
|
||||||
|
"registry_identity",
|
||||||
|
"git_ref",
|
||||||
|
)
|
||||||
|
if (text := _optional_str(raw, field))
|
||||||
|
}
|
||||||
|
if clean:
|
||||||
|
normalized[key] = clean
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def _parse_datetime(value: object) -> datetime | None:
|
def _parse_datetime(value: object) -> datetime | None:
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
@@ -628,8 +704,10 @@ def _invalid_catalog_result(
|
|||||||
channel: str | None,
|
channel: str | None,
|
||||||
sequence: int | None,
|
sequence: int | None,
|
||||||
generated_at: str | None,
|
generated_at: str | None,
|
||||||
|
not_before: str | None,
|
||||||
expires_at: str | None,
|
expires_at: str | None,
|
||||||
signature_state: dict[str, object],
|
signature_state: dict[str, object],
|
||||||
|
read_state: dict[str, object],
|
||||||
error: str,
|
error: str,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
@@ -638,10 +716,13 @@ def _invalid_catalog_result(
|
|||||||
"path": str(source) if source is not None else None,
|
"path": str(source) if source is not None else None,
|
||||||
"source": str(source) if source is not None else None,
|
"source": str(source) if source is not None else None,
|
||||||
"source_type": _catalog_source_type(source),
|
"source_type": _catalog_source_type(source),
|
||||||
|
"cache_used": bool(read_state.get("cache_used")),
|
||||||
|
"cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None,
|
||||||
"modules": list(modules),
|
"modules": list(modules),
|
||||||
"channel": channel,
|
"channel": channel,
|
||||||
"sequence": sequence,
|
"sequence": sequence,
|
||||||
"generated_at": generated_at,
|
"generated_at": generated_at,
|
||||||
|
"not_before": not_before,
|
||||||
"expires_at": expires_at,
|
"expires_at": expires_at,
|
||||||
"signed": signature_state["signed"],
|
"signed": signature_state["signed"],
|
||||||
"trusted": signature_state["trusted"],
|
"trusted": signature_state["trusted"],
|
||||||
|
|||||||
@@ -8,6 +8,9 @@ if TYPE_CHECKING:
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
|
||||||
|
SUPPORTED_MANIFEST_CONTRACT_VERSION = "1"
|
||||||
|
SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1"
|
||||||
|
|
||||||
PermissionLevel = Literal["system", "tenant"]
|
PermissionLevel = Literal["system", "tenant"]
|
||||||
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
SubjectType = Literal["account", "membership", "group", "service_account", "tenant"]
|
||||||
|
|
||||||
@@ -129,6 +132,61 @@ class ModuleContext:
|
|||||||
data: Mapping[str, Any] = field(default_factory=dict)
|
data: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
DocumentationLayer = Literal["always", "configured", "available", "evidence"]
|
||||||
|
DocumentationLinkKind = Literal["runtime", "api", "repository", "wiki", "public"]
|
||||||
|
DocumentationType = Literal["admin", "user"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentationLink:
|
||||||
|
label: str
|
||||||
|
href: str
|
||||||
|
kind: DocumentationLinkKind = "runtime"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentationCondition:
|
||||||
|
required_modules: tuple[str, ...] = ()
|
||||||
|
any_modules: tuple[str, ...] = ()
|
||||||
|
missing_modules: tuple[str, ...] = ()
|
||||||
|
required_capabilities: tuple[str, ...] = ()
|
||||||
|
required_scopes: tuple[str, ...] = ()
|
||||||
|
any_scopes: tuple[str, ...] = ()
|
||||||
|
configuration_keys: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentationTopic:
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
summary: str
|
||||||
|
body: str = ""
|
||||||
|
layer: DocumentationLayer = "configured"
|
||||||
|
documentation_types: tuple[DocumentationType, ...] = ("admin",)
|
||||||
|
audience: tuple[str, ...] = ()
|
||||||
|
order: int = 100
|
||||||
|
conditions: tuple[DocumentationCondition, ...] = ()
|
||||||
|
links: tuple[DocumentationLink, ...] = ()
|
||||||
|
related_modules: tuple[str, ...] = ()
|
||||||
|
unlocks: tuple[str, ...] = ()
|
||||||
|
configuration_keys: tuple[str, ...] = ()
|
||||||
|
i18n_key: str | None = None
|
||||||
|
translations: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
|
||||||
|
source_module_id: str | None = None
|
||||||
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DocumentationContext:
|
||||||
|
registry: object
|
||||||
|
principal: object | None = None
|
||||||
|
settings: object | None = None
|
||||||
|
session: object | None = None
|
||||||
|
documentation_type: DocumentationType = "admin"
|
||||||
|
locale: str = "en"
|
||||||
|
data: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
class ResourceAclProvider(Protocol):
|
class ResourceAclProvider(Protocol):
|
||||||
resource_type: str
|
resource_type: str
|
||||||
|
|
||||||
@@ -146,6 +204,7 @@ TenantSummaryProvider = Callable[[object, str], Mapping[str, int]]
|
|||||||
DeleteVetoProvider = Callable[[object, str, str], None]
|
DeleteVetoProvider = Callable[[object, str, str], None]
|
||||||
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
RouteFactory = Callable[[ModuleContext], "APIRouter"]
|
||||||
CapabilityFactory = Callable[[ModuleContext], object]
|
CapabilityFactory = Callable[[ModuleContext], object]
|
||||||
|
DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]]
|
||||||
LifecycleHook = Callable[[ModuleContext], None]
|
LifecycleHook = Callable[[ModuleContext], None]
|
||||||
|
|
||||||
|
|
||||||
@@ -170,3 +229,5 @@ class ModuleManifest:
|
|||||||
compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility)
|
compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility)
|
||||||
on_activate: LifecycleHook | None = None
|
on_activate: LifecycleHook | None = None
|
||||||
on_deactivate: LifecycleHook | None = None
|
on_deactivate: LifecycleHook | None = None
|
||||||
|
documentation: tuple[DocumentationTopic, ...] = ()
|
||||||
|
documentation_providers: tuple[DocumentationProvider, ...] = ()
|
||||||
|
|||||||
85
src/govoplan_core/core/organizations.py
Normal file
85
src/govoplan_core/core/organizations.py
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
ORGANIZATIONS_MODULE_ID = "organizations"
|
||||||
|
CAPABILITY_ORGANIZATION_DIRECTORY = "organizations.directory"
|
||||||
|
|
||||||
|
OrganizationStatus = Literal["active", "inactive", "suspended"]
|
||||||
|
FunctionAssignmentSource = Literal["direct", "delegated", "acting_for", "directory", "governance", "system"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OrganizationUnitRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
slug: str
|
||||||
|
name: str
|
||||||
|
parent_id: str | None = None
|
||||||
|
description: str | None = None
|
||||||
|
status: OrganizationStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OrganizationFunctionRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
organization_unit_id: str
|
||||||
|
slug: str
|
||||||
|
name: str
|
||||||
|
description: str | None = None
|
||||||
|
delegable: bool = False
|
||||||
|
act_in_place_allowed: bool = False
|
||||||
|
status: OrganizationStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OrganizationFunctionAssignmentRef:
|
||||||
|
id: str
|
||||||
|
tenant_id: str
|
||||||
|
account_id: str
|
||||||
|
function_id: str
|
||||||
|
organization_unit_id: str
|
||||||
|
identity_id: str | None = None
|
||||||
|
applies_to_subunits: bool = False
|
||||||
|
source: FunctionAssignmentSource = "direct"
|
||||||
|
delegated_from_assignment_id: str | None = None
|
||||||
|
acting_for_account_id: str | None = None
|
||||||
|
valid_from: datetime | None = None
|
||||||
|
valid_until: datetime | None = None
|
||||||
|
status: OrganizationStatus = "active"
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class OrganizationDirectory(Protocol):
|
||||||
|
def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def organization_units_for_tenant(self, tenant_id: str) -> Sequence[OrganizationUnitRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_function(self, function_id: str) -> OrganizationFunctionRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def functions_for_organization_unit(
|
||||||
|
self,
|
||||||
|
organization_unit_id: str,
|
||||||
|
*,
|
||||||
|
include_subunits: bool = False,
|
||||||
|
) -> Sequence[OrganizationFunctionRef]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_function_assignment(self, assignment_id: str) -> OrganizationFunctionAssignmentRef | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def function_assignments_for_account(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
*,
|
||||||
|
tenant_id: str | None = None,
|
||||||
|
) -> Sequence[OrganizationFunctionAssignmentRef]:
|
||||||
|
...
|
||||||
70
src/govoplan_core/core/pagination.py
Normal file
70
src/govoplan_core/core/pagination.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from datetime import date, datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
KEYSET_CURSOR_PREFIX = "ks1:"
|
||||||
|
|
||||||
|
|
||||||
|
class KeysetCursorError(ValueError):
|
||||||
|
"""Raised when a keyset cursor cannot be decoded for the current query."""
|
||||||
|
|
||||||
|
|
||||||
|
def keyset_query_fingerprint(scope: str, params: Mapping[str, Any]) -> str:
|
||||||
|
payload = {"scope": scope, "params": _jsonable(params)}
|
||||||
|
raw = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||||
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
def encode_keyset_cursor(scope: str, *, fingerprint: str, values: Mapping[str, Any]) -> str:
|
||||||
|
payload = {
|
||||||
|
"v": 1,
|
||||||
|
"scope": scope,
|
||||||
|
"fingerprint": fingerprint,
|
||||||
|
"values": _jsonable(values),
|
||||||
|
}
|
||||||
|
raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
encoded = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||||
|
return f"{KEYSET_CURSOR_PREFIX}{encoded}"
|
||||||
|
|
||||||
|
|
||||||
|
def decode_keyset_cursor(scope: str, cursor: str | None, *, fingerprint: str | None = None) -> dict[str, Any] | None:
|
||||||
|
if cursor is None or not cursor.strip():
|
||||||
|
return None
|
||||||
|
raw_cursor = cursor.strip()
|
||||||
|
if not raw_cursor.startswith(KEYSET_CURSOR_PREFIX):
|
||||||
|
raise KeysetCursorError("Invalid pagination cursor")
|
||||||
|
encoded = raw_cursor[len(KEYSET_CURSOR_PREFIX):]
|
||||||
|
try:
|
||||||
|
padded = encoded + ("=" * (-len(encoded) % 4))
|
||||||
|
payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise KeysetCursorError("Invalid pagination cursor") from exc
|
||||||
|
if not isinstance(payload, dict) or payload.get("v") != 1:
|
||||||
|
raise KeysetCursorError("Invalid pagination cursor")
|
||||||
|
if payload.get("scope") != scope:
|
||||||
|
raise KeysetCursorError("Pagination cursor does not match this endpoint")
|
||||||
|
if fingerprint is not None and payload.get("fingerprint") != fingerprint:
|
||||||
|
raise KeysetCursorError("Pagination cursor does not match the current query")
|
||||||
|
values = payload.get("values")
|
||||||
|
if not isinstance(values, dict):
|
||||||
|
raise KeysetCursorError("Invalid pagination cursor")
|
||||||
|
return values
|
||||||
|
|
||||||
|
|
||||||
|
def _jsonable(value: Any) -> Any:
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return value.isoformat()
|
||||||
|
if isinstance(value, date):
|
||||||
|
return value.isoformat()
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return str(value)
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
return {str(key): _jsonable(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))}
|
||||||
|
if isinstance(value, (list, tuple, set)):
|
||||||
|
return [_jsonable(item) for item in value]
|
||||||
|
return value
|
||||||
124
src/govoplan_core/core/policy.py
Normal file
124
src/govoplan_core/core/policy.py
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Iterable, Literal, Mapping, cast
|
||||||
|
from urllib.parse import quote, unquote
|
||||||
|
|
||||||
|
PolicyScopeType = Literal["system", "tenant", "user", "group", "campaign"]
|
||||||
|
|
||||||
|
POLICY_SCOPE_TYPES: tuple[PolicyScopeType, ...] = ("system", "tenant", "user", "group", "campaign")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_policy_scope_type(scope_type: str) -> PolicyScopeType:
|
||||||
|
clean = scope_type.strip().casefold()
|
||||||
|
if clean not in POLICY_SCOPE_TYPES:
|
||||||
|
raise ValueError("Policy scope must be system, tenant, user, group or campaign")
|
||||||
|
return cast(PolicyScopeType, clean)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PolicySourceRef:
|
||||||
|
scope_type: PolicyScopeType
|
||||||
|
scope_id: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self) -> str:
|
||||||
|
return policy_source_path(self.scope_type, self.scope_id)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {"scope_type": self.scope_type, "scope_id": self.scope_id, "path": self.path}
|
||||||
|
|
||||||
|
|
||||||
|
def policy_source_path(scope_type: str, scope_id: str | None = None) -> str:
|
||||||
|
clean_scope = normalize_policy_scope_type(scope_type)
|
||||||
|
if clean_scope == "system":
|
||||||
|
if scope_id:
|
||||||
|
raise ValueError("System policy sources do not carry a scope_id")
|
||||||
|
return "system"
|
||||||
|
if not scope_id:
|
||||||
|
raise ValueError(f"{clean_scope.capitalize()} policy sources require a scope_id")
|
||||||
|
return f"{clean_scope}:{quote(str(scope_id), safe='')}"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_policy_source_path(path: str) -> PolicySourceRef:
|
||||||
|
clean_path = path.strip()
|
||||||
|
if clean_path == "system":
|
||||||
|
return PolicySourceRef(scope_type="system")
|
||||||
|
scope_type, separator, encoded_scope_id = clean_path.partition(":")
|
||||||
|
if not separator:
|
||||||
|
raise ValueError("Policy source path must be system or <scope_type>:<url-encoded-scope-id>")
|
||||||
|
clean_scope = normalize_policy_scope_type(scope_type)
|
||||||
|
if clean_scope == "system":
|
||||||
|
raise ValueError("System policy source path must be exactly system")
|
||||||
|
scope_id = unquote(encoded_scope_id)
|
||||||
|
if not scope_id:
|
||||||
|
raise ValueError("Policy source path requires a non-empty scope id")
|
||||||
|
return PolicySourceRef(scope_type=clean_scope, scope_id=scope_id)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PolicySourceStep:
|
||||||
|
scope_type: PolicyScopeType
|
||||||
|
label: str
|
||||||
|
scope_id: str | None = None
|
||||||
|
applied_fields: tuple[str, ...] = ()
|
||||||
|
policy: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self) -> str:
|
||||||
|
return policy_source_path(self.scope_type, self.scope_id)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"scope_type": self.scope_type,
|
||||||
|
"scope_id": self.scope_id,
|
||||||
|
"path": self.path,
|
||||||
|
"label": self.label,
|
||||||
|
"applied_fields": list(self.applied_fields),
|
||||||
|
"policy": dict(self.policy),
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> PolicySourceStep:
|
||||||
|
policy_value = value.get("policy")
|
||||||
|
return cls(
|
||||||
|
scope_type=normalize_policy_scope_type(str(value.get("scope_type", ""))),
|
||||||
|
scope_id=str(value["scope_id"]) if value.get("scope_id") is not None else None,
|
||||||
|
label=str(value.get("label") or ""),
|
||||||
|
applied_fields=tuple(str(field) for field in (value.get("applied_fields") or ())),
|
||||||
|
policy=policy_value if isinstance(policy_value, Mapping) else {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def policy_source_step(
|
||||||
|
scope_type: str,
|
||||||
|
label: str,
|
||||||
|
scope_id: str | None,
|
||||||
|
applied_fields: Iterable[str] = (),
|
||||||
|
policy: Mapping[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return PolicySourceStep(
|
||||||
|
scope_type=normalize_policy_scope_type(scope_type),
|
||||||
|
scope_id=scope_id,
|
||||||
|
label=label,
|
||||||
|
applied_fields=tuple(str(field) for field in applied_fields),
|
||||||
|
policy=dict(policy or {}),
|
||||||
|
).to_dict()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PolicyDecision:
|
||||||
|
allowed: bool
|
||||||
|
reason: str | None = None
|
||||||
|
source_path: tuple[PolicySourceStep, ...] = ()
|
||||||
|
requirements: tuple[str, ...] = ()
|
||||||
|
details: Mapping[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"allowed": self.allowed,
|
||||||
|
"reason": self.reason,
|
||||||
|
"source_path": [step.to_dict() for step in self.source_path],
|
||||||
|
"requirements": list(self.requirements),
|
||||||
|
"details": dict(self.details),
|
||||||
|
}
|
||||||
@@ -14,9 +14,13 @@ from govoplan_core.core.modules import (
|
|||||||
PermissionDefinition,
|
PermissionDefinition,
|
||||||
ResourceAclProvider,
|
ResourceAclProvider,
|
||||||
RoleTemplate,
|
RoleTemplate,
|
||||||
|
SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION,
|
||||||
|
SUPPORTED_MANIFEST_CONTRACT_VERSION,
|
||||||
TenantSummaryProvider,
|
TenantSummaryProvider,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_MODULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]*$")
|
||||||
|
_NPM_PACKAGE_RE = re.compile(r"^(?:@[a-z0-9][a-z0-9_.-]*/)?[a-z0-9][a-z0-9_.-]*$")
|
||||||
_SCOPE_RE = re.compile(r"^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$")
|
_SCOPE_RE = re.compile(r"^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$")
|
||||||
_WILDCARD_RE = re.compile(r"^([a-z][a-z0-9_]*|\*):\*$|^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:\*$")
|
_WILDCARD_RE = re.compile(r"^([a-z][a-z0-9_]*|\*):\*$|^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:\*$")
|
||||||
|
|
||||||
@@ -150,6 +154,7 @@ class PlatformRegistry:
|
|||||||
ordered = tuple(self._topologically_sorted())
|
ordered = tuple(self._topologically_sorted())
|
||||||
seen_permissions: dict[str, PermissionDefinition] = {}
|
seen_permissions: dict[str, PermissionDefinition] = {}
|
||||||
for manifest in ordered:
|
for manifest in ordered:
|
||||||
|
_validate_manifest_shape(manifest)
|
||||||
for dependency in manifest.dependencies:
|
for dependency in manifest.dependencies:
|
||||||
if dependency not in self._manifests:
|
if dependency not in self._manifests:
|
||||||
raise RegistryError(f"Module {manifest.id!r} depends on unknown module {dependency!r}")
|
raise RegistryError(f"Module {manifest.id!r} depends on unknown module {dependency!r}")
|
||||||
@@ -207,3 +212,73 @@ class PlatformRegistry:
|
|||||||
unresolved = ", ".join(sorted(module_id for module_id, dependencies in incoming.items() if dependencies))
|
unresolved = ", ".join(sorted(module_id for module_id, dependencies in incoming.items() if dependencies))
|
||||||
raise RegistryError(f"Module dependency cycle or unresolved dependency: {unresolved}")
|
raise RegistryError(f"Module dependency cycle or unresolved dependency: {unresolved}")
|
||||||
return (self._manifests[module_id] for module_id in ordered)
|
return (self._manifests[module_id] for module_id in ordered)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_manifest_shape(manifest: ModuleManifest) -> None:
|
||||||
|
if not _MODULE_ID_RE.match(manifest.id):
|
||||||
|
raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}")
|
||||||
|
if not manifest.name.strip():
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} must declare a non-empty name")
|
||||||
|
if not manifest.version.strip():
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} must declare a non-empty version")
|
||||||
|
if manifest.compatibility.manifest_contract_version != SUPPORTED_MANIFEST_CONTRACT_VERSION:
|
||||||
|
raise RegistryError(
|
||||||
|
f"Module {manifest.id!r} uses unsupported manifest contract version "
|
||||||
|
f"{manifest.compatibility.manifest_contract_version!r}; supported version is "
|
||||||
|
f"{SUPPORTED_MANIFEST_CONTRACT_VERSION!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
_validate_dependency_list(manifest.id, "dependencies", manifest.dependencies)
|
||||||
|
_validate_dependency_list(manifest.id, "optional_dependencies", manifest.optional_dependencies)
|
||||||
|
overlap = set(manifest.dependencies) & set(manifest.optional_dependencies)
|
||||||
|
if overlap:
|
||||||
|
joined = ", ".join(sorted(overlap))
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} lists dependencies as both required and optional: {joined}")
|
||||||
|
|
||||||
|
if manifest.migration_spec is not None:
|
||||||
|
if manifest.migration_spec.module_id != manifest.id:
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} has migration spec for {manifest.migration_spec.module_id!r}")
|
||||||
|
if manifest.migration_spec.metadata is None and not manifest.migration_spec.script_location:
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} migration spec must declare metadata or script location")
|
||||||
|
|
||||||
|
if manifest.frontend is not None:
|
||||||
|
frontend = manifest.frontend
|
||||||
|
if frontend.module_id != manifest.id:
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} has frontend metadata for {frontend.module_id!r}")
|
||||||
|
if frontend.asset_manifest_contract_version != SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION:
|
||||||
|
raise RegistryError(
|
||||||
|
f"Module {manifest.id!r} uses unsupported frontend asset manifest contract version "
|
||||||
|
f"{frontend.asset_manifest_contract_version!r}; supported version is "
|
||||||
|
f"{SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION!r}"
|
||||||
|
)
|
||||||
|
if frontend.package_name is not None and not _NPM_PACKAGE_RE.match(frontend.package_name):
|
||||||
|
raise RegistryError(f"Module {manifest.id!r} has invalid frontend package name {frontend.package_name!r}")
|
||||||
|
for route in (*frontend.routes, *frontend.settings_routes):
|
||||||
|
if not route.path.startswith("/"):
|
||||||
|
raise RegistryError(f"Frontend route for module {manifest.id!r} must start with '/': {route.path!r}")
|
||||||
|
if not route.component.strip():
|
||||||
|
raise RegistryError(f"Frontend route {route.path!r} for module {manifest.id!r} must declare a component")
|
||||||
|
for item in frontend.nav_items:
|
||||||
|
_validate_nav_item(manifest.id, item)
|
||||||
|
|
||||||
|
for item in manifest.nav_items:
|
||||||
|
_validate_nav_item(manifest.id, item)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_dependency_list(module_id: str, field_name: str, dependencies: tuple[str, ...]) -> None:
|
||||||
|
if len(dependencies) != len(set(dependencies)):
|
||||||
|
raise RegistryError(f"Module {module_id!r} has duplicate {field_name}")
|
||||||
|
for dependency in dependencies:
|
||||||
|
if dependency == module_id:
|
||||||
|
raise RegistryError(f"Module {module_id!r} cannot depend on itself")
|
||||||
|
if not _MODULE_ID_RE.match(dependency):
|
||||||
|
raise RegistryError(f"Module {module_id!r} has invalid dependency id {dependency!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_nav_item(module_id: str, item: NavItem) -> None:
|
||||||
|
if not item.path.startswith("/"):
|
||||||
|
raise RegistryError(f"Navigation item for module {module_id!r} must start with '/': {item.path!r}")
|
||||||
|
if not item.label.strip():
|
||||||
|
raise RegistryError(f"Navigation item {item.path!r} for module {module_id!r} must declare a label")
|
||||||
|
if item.icon is not None and not item.icon.strip():
|
||||||
|
raise RegistryError(f"Navigation item {item.path!r} for module {module_id!r} has an empty icon name")
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ def create_all_tables() -> None:
|
|||||||
# Build the configured registry so enabled module manifests register their
|
# Build the configured registry so enabled module manifests register their
|
||||||
# model metadata with the shared SQLAlchemy base before create_all runs.
|
# model metadata with the shared SQLAlchemy base before create_all runs.
|
||||||
from govoplan_core.admin import models as core_admin_models # noqa: F401
|
from govoplan_core.admin import models as core_admin_models # noqa: F401
|
||||||
|
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401
|
||||||
|
|
||||||
raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules)
|
raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules)
|
||||||
candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules)
|
candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules)
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from alembic.runtime.migration import MigrationContext
|
|||||||
from sqlalchemy import create_engine, inspect, text
|
from sqlalchemy import create_engine, inspect, text
|
||||||
|
|
||||||
from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan
|
from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan
|
||||||
|
from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata
|
||||||
|
from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor
|
||||||
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids
|
||||||
from govoplan_core.db.session import configure_database
|
from govoplan_core.db.session import configure_database
|
||||||
from govoplan_core.server.default_config import get_server_config
|
from govoplan_core.server.default_config import get_server_config
|
||||||
@@ -26,8 +28,27 @@ from govoplan_core.settings import settings
|
|||||||
REVISION_AUTH_RBAC = "2c3d4e5f6a7b"
|
REVISION_AUTH_RBAC = "2c3d4e5f6a7b"
|
||||||
REVISION_FILE_STORAGE = "3d4e5f6a7b8c"
|
REVISION_FILE_STORAGE = "3d4e5f6a7b8c"
|
||||||
REVISION_FILE_FOLDERS = "4e5f6a7b8c9d"
|
REVISION_FILE_FOLDERS = "4e5f6a7b8c9d"
|
||||||
|
REVISION_NAMESPACE_PLATFORM_TABLES = "2e3f4a5b6c7d"
|
||||||
|
REVISION_CORE_CHANGE_SEQUENCE = "3f4a5b6c7d8e"
|
||||||
REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0"
|
REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0"
|
||||||
|
|
||||||
|
_NAMESPACE_TABLE_RENAMES = (
|
||||||
|
("tenants", "tenancy_tenants"),
|
||||||
|
("accounts", "access_accounts"),
|
||||||
|
("users", "access_users"),
|
||||||
|
("groups", "access_groups"),
|
||||||
|
("roles", "access_roles"),
|
||||||
|
("system_role_assignments", "access_system_role_assignments"),
|
||||||
|
("user_group_memberships", "access_user_group_memberships"),
|
||||||
|
("user_role_assignments", "access_user_role_assignments"),
|
||||||
|
("group_role_assignments", "access_group_role_assignments"),
|
||||||
|
("api_keys", "access_api_keys"),
|
||||||
|
("auth_sessions", "access_auth_sessions"),
|
||||||
|
("system_settings", "core_system_settings"),
|
||||||
|
("governance_templates", "admin_governance_templates"),
|
||||||
|
("governance_template_assignments", "admin_governance_template_assignments"),
|
||||||
|
)
|
||||||
|
|
||||||
_FILE_STORAGE_TABLES = {
|
_FILE_STORAGE_TABLES = {
|
||||||
"file_blobs",
|
"file_blobs",
|
||||||
"file_assets",
|
"file_assets",
|
||||||
@@ -74,21 +95,21 @@ _FILE_STORAGE_COLUMNS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_CREATE_ALL_THROUGH_HIERARCHICAL_TABLES = {
|
_CREATE_ALL_THROUGH_HIERARCHICAL_TABLES = {
|
||||||
"accounts",
|
"access_accounts",
|
||||||
"auth_sessions",
|
"access_auth_sessions",
|
||||||
"audit_log",
|
"audit_log",
|
||||||
"campaign_versions",
|
"campaign_versions",
|
||||||
"campaign_jobs",
|
"campaign_jobs",
|
||||||
"send_attempts",
|
"send_attempts",
|
||||||
"system_role_assignments",
|
"access_system_role_assignments",
|
||||||
"system_settings",
|
"core_system_settings",
|
||||||
"governance_templates",
|
"admin_governance_templates",
|
||||||
"governance_template_assignments",
|
"admin_governance_template_assignments",
|
||||||
"mail_server_profiles",
|
"mail_server_profiles",
|
||||||
}
|
}
|
||||||
|
|
||||||
_CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = {
|
_CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = {
|
||||||
"auth_sessions": {"account_id", "csrf_token_hash"},
|
"access_auth_sessions": {"account_id", "csrf_token_hash"},
|
||||||
"audit_log": {"scope", "tenant_id", "user_id", "api_key_id"},
|
"audit_log": {"scope", "tenant_id", "user_id", "api_key_id"},
|
||||||
"campaign_versions": {
|
"campaign_versions": {
|
||||||
"workflow_state",
|
"workflow_state",
|
||||||
@@ -102,13 +123,13 @@ _CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = {
|
|||||||
},
|
},
|
||||||
"campaign_jobs": {"claimed_at", "claim_token", "smtp_started_at", "outcome_unknown_at", "eml_sha256"},
|
"campaign_jobs": {"claimed_at", "claim_token", "smtp_started_at", "outcome_unknown_at", "eml_sha256"},
|
||||||
"send_attempts": {"status", "claim_token"},
|
"send_attempts": {"status", "claim_token"},
|
||||||
"users": {"account_id", "settings", "mail_profile_policy"},
|
"access_users": {"account_id", "settings", "mail_profile_policy"},
|
||||||
"groups": {"system_template_id", "system_required", "settings", "mail_profile_policy"},
|
"access_groups": {"system_template_id", "system_required", "settings", "mail_profile_policy"},
|
||||||
"roles": {"system_template_id", "system_required"},
|
"access_roles": {"system_template_id", "system_required"},
|
||||||
"tenants": {"settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"},
|
"tenancy_tenants": {"settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"},
|
||||||
"campaigns": {"settings", "mail_profile_policy"},
|
"campaigns": {"settings", "mail_profile_policy"},
|
||||||
"mail_server_profiles": {"scope_type", "scope_id"},
|
"mail_server_profiles": {"scope_type", "scope_id"},
|
||||||
"system_settings": {"settings", "allow_tenant_custom_groups", "allow_tenant_custom_roles", "allow_tenant_api_keys"},
|
"core_system_settings": {"settings", "allow_tenant_custom_groups", "allow_tenant_custom_roles", "allow_tenant_api_keys"},
|
||||||
}
|
}
|
||||||
|
|
||||||
_CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = {
|
_CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = {
|
||||||
@@ -121,10 +142,10 @@ _CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = {
|
|||||||
"campaign_jobs": {"ix_campaign_jobs_claim_token", "ix_campaign_jobs_eml_sha256"},
|
"campaign_jobs": {"ix_campaign_jobs_claim_token", "ix_campaign_jobs_eml_sha256"},
|
||||||
"send_attempts": {"ix_send_attempts_status", "ix_send_attempts_claim_token"},
|
"send_attempts": {"ix_send_attempts_status", "ix_send_attempts_claim_token"},
|
||||||
"audit_log": {"ix_audit_log_scope_created_at", "ix_audit_log_tenant_scope_created_at"},
|
"audit_log": {"ix_audit_log_scope_created_at", "ix_audit_log_tenant_scope_created_at"},
|
||||||
"auth_sessions": {"ix_auth_sessions_account_id"},
|
"access_auth_sessions": {"ix_access_auth_sessions_account_id"},
|
||||||
"users": {"ix_users_account_id"},
|
"access_users": {"ix_access_users_account_id"},
|
||||||
"groups": {"ix_groups_system_template_id"},
|
"access_groups": {"ix_access_groups_system_template_id"},
|
||||||
"roles": {"ix_roles_system_template_id"},
|
"access_roles": {"ix_access_roles_system_template_id"},
|
||||||
"mail_server_profiles": {"ix_mail_server_profiles_scope_type", "ix_mail_server_profiles_scope_id", "ix_mail_server_profiles_scope"},
|
"mail_server_profiles": {"ix_mail_server_profiles_scope_type", "ix_mail_server_profiles_scope_id", "ix_mail_server_profiles_scope"},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -258,6 +279,111 @@ def _backfill_user_lock_state_for_create_all_schema(database_url: str) -> None:
|
|||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def _row_count(connection, table_name: str) -> int:
|
||||||
|
quoted = connection.dialect.identifier_preparer.quote(table_name)
|
||||||
|
return int(connection.execute(text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_table(connection, table_name: str) -> None:
|
||||||
|
quoted = connection.dialect.identifier_preparer.quote(table_name)
|
||||||
|
connection.execute(text(f"DROP TABLE {quoted}"))
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_table(connection, old_name: str, new_name: str) -> None:
|
||||||
|
quoted_old = connection.dialect.identifier_preparer.quote(old_name)
|
||||||
|
quoted_new = connection.dialect.identifier_preparer.quote(new_name)
|
||||||
|
connection.execute(text(f"ALTER TABLE {quoted_old} RENAME TO {quoted_new}"))
|
||||||
|
|
||||||
|
|
||||||
|
def reconcile_namespace_table_drift(database_url: str | None = None) -> bool:
|
||||||
|
"""Repair dev databases stamped past the namespace-table migration.
|
||||||
|
|
||||||
|
During the repository split some development databases were stamped at the
|
||||||
|
newer Alembic heads while still carrying the old platform table names. The
|
||||||
|
real migration only renames tables, so replay that idempotent operation
|
||||||
|
before startup bootstrap code touches the ORM.
|
||||||
|
"""
|
||||||
|
|
||||||
|
url = database_url or settings.database_url
|
||||||
|
changed = False
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
with engine.begin() as connection:
|
||||||
|
schema = inspect(connection)
|
||||||
|
tables = set(schema.get_table_names())
|
||||||
|
if not any(old_name in tables and new_name not in tables for old_name, new_name in _NAMESPACE_TABLE_RENAMES):
|
||||||
|
return False
|
||||||
|
|
||||||
|
old_tables_to_drop: set[str] = set()
|
||||||
|
new_tables_to_drop: set[str] = set()
|
||||||
|
for old_name, new_name in _NAMESPACE_TABLE_RENAMES:
|
||||||
|
if old_name not in tables or new_name not in tables:
|
||||||
|
continue
|
||||||
|
if _row_count(connection, old_name) == 0:
|
||||||
|
old_tables_to_drop.add(old_name)
|
||||||
|
elif _row_count(connection, new_name) == 0:
|
||||||
|
new_tables_to_drop.add(new_name)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Cannot reconcile non-empty {old_name} over non-empty {new_name}")
|
||||||
|
|
||||||
|
for old_name, _new_name in reversed(_NAMESPACE_TABLE_RENAMES):
|
||||||
|
if old_name in old_tables_to_drop:
|
||||||
|
_drop_table(connection, old_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
changed = True
|
||||||
|
for _old_name, new_name in reversed(_NAMESPACE_TABLE_RENAMES):
|
||||||
|
if new_name in new_tables_to_drop:
|
||||||
|
_drop_table(connection, new_name)
|
||||||
|
tables.remove(new_name)
|
||||||
|
changed = True
|
||||||
|
|
||||||
|
for old_name, new_name in _NAMESPACE_TABLE_RENAMES:
|
||||||
|
if old_name not in tables or new_name in tables:
|
||||||
|
continue
|
||||||
|
_rename_table(connection, old_name, new_name)
|
||||||
|
tables.remove(old_name)
|
||||||
|
tables.add(new_name)
|
||||||
|
changed = True
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def reconcile_change_sequence_retention_floor_drift(database_url: str | None = None) -> bool:
|
||||||
|
"""Repair databases stamped after the change-sequence migration changed.
|
||||||
|
|
||||||
|
Early development databases may have applied the change-sequence revision
|
||||||
|
before the retention-floor table was added to that migration. The main
|
||||||
|
change-sequence table is enough for full snapshots, but incremental delta
|
||||||
|
requests need the retention floor to decide whether a watermark is stale.
|
||||||
|
"""
|
||||||
|
|
||||||
|
url = database_url or settings.database_url
|
||||||
|
changed = False
|
||||||
|
engine = create_engine(url)
|
||||||
|
try:
|
||||||
|
with engine.begin() as connection:
|
||||||
|
schema = inspect(connection)
|
||||||
|
tables = set(schema.get_table_names())
|
||||||
|
if ChangeSequenceEntry.__tablename__ not in tables:
|
||||||
|
return False
|
||||||
|
if ChangeSequenceRetentionFloor.__tablename__ not in tables:
|
||||||
|
ChangeSequenceRetentionFloor.__table__.create(bind=connection, checkfirst=True)
|
||||||
|
changed = True
|
||||||
|
else:
|
||||||
|
indexes = {
|
||||||
|
index["name"]
|
||||||
|
for index in schema.get_indexes(ChangeSequenceRetentionFloor.__tablename__)
|
||||||
|
}
|
||||||
|
for index in ChangeSequenceRetentionFloor.__table__.indexes:
|
||||||
|
if index.name not in indexes:
|
||||||
|
index.create(bind=connection)
|
||||||
|
changed = True
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None:
|
def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None:
|
||||||
"""Repair the known Alembic/create_all drift without modifying application data.
|
"""Repair the known Alembic/create_all drift without modifying application data.
|
||||||
|
|
||||||
@@ -327,6 +453,9 @@ def migrate_database(
|
|||||||
manifest_factories: tuple[ManifestFactory, ...] = (),
|
manifest_factories: tuple[ManifestFactory, ...] = (),
|
||||||
) -> MigrationResult:
|
) -> MigrationResult:
|
||||||
url = database_url or settings.database_url
|
url = database_url or settings.database_url
|
||||||
|
if reconcile_legacy_schema:
|
||||||
|
reconcile_namespace_table_drift(url)
|
||||||
|
reconcile_change_sequence_retention_floor_drift(url)
|
||||||
previous = database_revision(url)
|
previous = database_revision(url)
|
||||||
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
|
reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None
|
||||||
command.upgrade(
|
command.upgrade(
|
||||||
|
|||||||
@@ -38,15 +38,21 @@ class DatabaseHandle:
|
|||||||
with self.SessionLocal() as session:
|
with self.SessionLocal() as session:
|
||||||
yield session
|
yield session
|
||||||
|
|
||||||
|
def dispose(self) -> None:
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
_default_database: DatabaseHandle | None = None
|
_default_database: DatabaseHandle | None = None
|
||||||
|
|
||||||
|
|
||||||
def configure_database(database_url: str, *, engine: Engine | None = None) -> DatabaseHandle:
|
def configure_database(database_url: str, *, engine: Engine | None = None, dispose_previous: bool = False) -> DatabaseHandle:
|
||||||
global _default_database
|
global _default_database
|
||||||
if engine is None and _default_database is not None and _default_database.database_url == database_url:
|
if engine is None and _default_database is not None and _default_database.database_url == database_url:
|
||||||
return _default_database
|
return _default_database
|
||||||
|
previous_database = _default_database
|
||||||
_default_database = DatabaseHandle(database_url, engine=engine)
|
_default_database = DatabaseHandle(database_url, engine=engine)
|
||||||
|
if dispose_previous and previous_database is not None and previous_database is not _default_database:
|
||||||
|
previous_database.dispose()
|
||||||
return _default_database
|
return _default_database
|
||||||
|
|
||||||
|
|
||||||
@@ -56,6 +62,13 @@ def set_database(handle: DatabaseHandle) -> DatabaseHandle:
|
|||||||
return handle
|
return handle
|
||||||
|
|
||||||
|
|
||||||
|
def reset_database(*, dispose: bool = False) -> None:
|
||||||
|
global _default_database
|
||||||
|
if dispose and _default_database is not None:
|
||||||
|
_default_database.dispose()
|
||||||
|
_default_database = None
|
||||||
|
|
||||||
|
|
||||||
def get_database() -> DatabaseHandle:
|
def get_database() -> DatabaseHandle:
|
||||||
if _default_database is None:
|
if _default_database is None:
|
||||||
raise RuntimeError("GovOPlaN database is not configured")
|
raise RuntimeError("GovOPlaN database is not configured")
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user